1 /***********************************************************************
2  * Copyright (c) 2013-2024 Commonwealth Computer Research, Inc.
3  * All rights reserved. This program and the accompanying materials
4  * are made available under the terms of the Apache License, Version 2.0
5  * which accompanies this distribution and is available at
6  * http://www.opensource.org/licenses/apache2.0.php.
7  ***********************************************************************/
8 
9 package org.locationtech.geomesa.utils.stats
10 
11 import com.google.gson._
12 import org.apache.commons.text.StringEscapeUtils
13 import org.geotools.api.feature.simple.{SimpleFeature, SimpleFeatureType}
14 import org.locationtech.geomesa.curve.TimePeriod.TimePeriod
15 import org.locationtech.geomesa.utils.date.DateUtils.toInstant
16 import org.locationtech.geomesa.utils.geotools._
17 import org.locationtech.geomesa.utils.text.WKTUtils
18 import org.locationtech.jts.geom.Geometry
19 
20 import java.lang.reflect.Type
21 import java.lang.{Double => jDouble, Float => jFloat, Long => jLong}
22 import java.time.{LocalDateTime, ZoneOffset}
23 import java.util.Date
24 import scala.collection.JavaConverters._
25 import scala.reflect.ClassTag
26 
27 /**
28  * Stats used by the StatsIterator to compute various statistics server-side for a given query.
29  */
30 trait Stat {
31 
32   type S <: Stat
33 
34   /**
35     * The simple feature type that this stat operates on
36     *
37     * @return
38     */
39   def sft: SimpleFeatureType
40 
41   /**
42    * Compute statistics based upon the given simple feature.
43    * This method will be called for every SimpleFeature a query returns.
44    *
45    * @param sf feature to evaluate
46    */
47   def observe(sf: SimpleFeature): Unit
48 
49   /**
50     * Tries to remove the given simple feature from the compiled statistics.
51     * Note: may not be possible to un-observe a feature, in which case this method will
52     * have no effect.
53     *
54     * @param sf feature to un-evaluate
55     */
56   def unobserve(sf: SimpleFeature): Unit
57 
58   /**
59    * Add another stat to this stat. Avoids allocating another object.
60    *
61    * @param other the other stat to add
62    */
63   def +=(other: S): Unit
64 
65   /**
66    * Non type-safe add - if stats are not the same type, will throw an exception
67    *
68    * @param other the other stat to add
69    */
70   def +=(other: Stat)(implicit d: DummyImplicit): Unit = this += other.asInstanceOf[S]
71 
72   /**
73     * Combine two stats into a new stat
74     *
75     * @param other the other stat to add
76     */
77   def +(other: S): S
78 
79   /**
80     * Non type-safe add - if stats are not the same type, will throw an exception
81     *
82     * @param other the other stat to add
83     */
84   def +(other: Stat)(implicit d: DummyImplicit): Stat = this + other.asInstanceOf[S]
85 
86   /**
87    * Returns a JSON representation of the [[Stat]]
88    *
89    * @return stat as a json string
90    */
91   def toJson: String = Stat.JSON.toJson(toJsonObject)
92 
93   /**
94     * Returns a representation of the [[Stat]] to be serialized
95     *
96     * This function should return a representation (view) of the [[Stat]] to be serialized as JSON.
97     * Instances of [[Map]] can be used to represent JSON dictionaries or [[Seq]] for JSON arrays.
98     * A [[collection.SortedMap]] such as [[collection.immutable.ListMap]] is recommended if key order
99     * should be deterministic.  Other types may be used but could require the creation and registration
100     * of custom serializers dependent on the JSON framework being utilized (currently [[Gson]]).
101     *
102     * @return stat as a json serializable object
103     */
104   def toJsonObject: Any
105 
106   /**
107    * Necessary method used by the StatIterator. Indicates if the stat has any values or not
108    *
109    * @return true if stat contains values
110    */
111   def isEmpty: Boolean
112 
113   /**
114     * Compares the two stats for equivalence. We don't use standard 'equals' as it gets messy with
115     * mutable state and hash codes
116     *
117     * @param other other stat to compare
118     * @return true if equals
119     */
120   def isEquivalent(other: Stat): Boolean
121 
122   /**
123    * Clears the stat to its original state when first initialized.
124    * Necessary method used by the StatIterator.
125    */
126   def clear(): Unit
127 }
128 
129 /**
130  * This class contains parsers which dictate how to instantiate a particular Stat.
131  * Stats are created by passing a stats string as a query hint (QueryHints.STATS_STRING).
132  *
133  * A valid stats string should adhere to the parsers here:
134  * e.g. "MinMax(attributeName);IteratorCount" or "RangeHistogram(attributeName,10,0,100)"
135  * (see tests for more use cases)
136  */
137 object Stat {
138 
139   private val ScalaMapSerializer = new JsonSerializer[Map[Any,Any]] {
140     def serialize(s: Map[Any,Any], t: Type, jsc: JsonSerializationContext): JsonElement = jsc.serialize(s.asJava)
141   }
142   private val ScalaSeqSerializer = new JsonSerializer[Seq[Any]] {
143     def serialize(s: Seq[Any], t: Type, jsc: JsonSerializationContext): JsonElement = jsc.serialize(s.asJava)
144   }
145   private val StatSerializer = new JsonSerializer[Stat] {
146     def serialize(s: Stat, t: Type, jsc: JsonSerializationContext): JsonElement = jsc.serialize(s.toJsonObject)
147   }
148   private val GeometrySerializer = new JsonSerializer[Geometry] {
149     def serialize(g: Geometry, t: Type, jsc: JsonSerializationContext): JsonElement =
150       new JsonPrimitive(WKTUtils.write(g))
151   }
152   private val DateSerializer = new JsonSerializer[Date] {
153     def serialize(d: Date, t: Type, jsc: JsonSerializationContext): JsonElement =
154       new JsonPrimitive(GeoToolsDateFormat.format(toInstant(d)))
155   }
156   private val DoubleSerializer = new JsonSerializer[jDouble]() {
157     def serialize(double: jDouble, t: Type, jsc: JsonSerializationContext): JsonElement = double match {
158       /* NaN check, use null to mirror existing behavior for missing/invalid values */
159       case d if jDouble.isNaN(d) => JsonNull.INSTANCE
160       case d if d == jDouble.NEGATIVE_INFINITY => new JsonPrimitive("Infinity")
161       case d if d == jDouble.POSITIVE_INFINITY => new JsonPrimitive("+Infinity")
162       case _ => new JsonPrimitive(double)
163     }
164   }
165   private val FloatSerializer = new JsonSerializer[jFloat]() {
166     def serialize(float: jFloat, t: Type, jsc: JsonSerializationContext): JsonElement = float match {
167       /* NaN check, use null to mirror existing behavior for missing/invalid values */
168       case f if jFloat.isNaN(f) => JsonNull.INSTANCE
169       case f if f == jFloat.NEGATIVE_INFINITY => new JsonPrimitive("Infinity")
170       case f if f == jFloat.POSITIVE_INFINITY => new JsonPrimitive("+Infinity")
171       case _ => new JsonPrimitive(float)
172     }
173   }
174 
175   private val JSON: Gson = new GsonBuilder()
176     .serializeNulls()
177     .registerTypeAdapter(classOf[Double], DoubleSerializer)
178     .registerTypeAdapter(classOf[jDouble], DoubleSerializer)
179     .registerTypeAdapter(classOf[Float], FloatSerializer)
180     .registerTypeAdapter(classOf[jFloat], FloatSerializer)
181     .registerTypeHierarchyAdapter(classOf[Stat], StatSerializer)
182     .registerTypeHierarchyAdapter(classOf[Geometry], GeometrySerializer)
183     .registerTypeHierarchyAdapter(classOf[Date], DateSerializer)
184     .registerTypeHierarchyAdapter(classOf[Map[_,_]], ScalaMapSerializer)
185     .registerTypeHierarchyAdapter(classOf[Seq[_]], ScalaSeqSerializer)
186     .create()
187 
188   def apply(sft: SimpleFeatureType, s: String): Stat = StatParser.parse(sft, s)
189 
190   /**
191     * String that will be parsed to a count stat
192     *
193     * @return
194     */
195   def Count(): String = "Count()"
196 
197   /**
198     * String that will be parsed to a min/max stat
199     *
200     * @param attribute attribute name to min/max
201     * @return
202     */
203   def MinMax(attribute: String): String = s"MinMax(${safeString(attribute)})"
204 
205   /**
206     * String that will be parsed to a histogram stat
207     *
208     * @param attribute attribute name to histogram
209     * @return
210     */
211   def Enumeration(attribute: String): String = s"Enumeration(${safeString(attribute)})"
212 
213   /**
214     * String that will be parsed into a TopK stat
215     *
216     * @param attribute attribute name to evaluate
217     * @return
218     */
219   def TopK(attribute: String): String = s"TopK(${safeString(attribute)})"
220 
221   /**
222     * String that will be parsed into a count min sketch stat
223     *
224     * @param attribute attribute to sketch
225     * @param precision precision of the sketch - @see Frequency
226     * @return
227     */
228   def Frequency(attribute: String, precision: Int): String =
229     s"Frequency(${safeString(attribute)},$precision)"
230 
231   /**
232     * String that will be parsed into a count min sketch stat
233     *
234     * @param attribute attribute to sketch
235     * @param dtg date attribute to use for binning
236     * @param period time period to split on
237     * @param precision precision of the sketch - @see Frequency
238     * @return
239     */
240   def Frequency(attribute: String, dtg: String, period: TimePeriod, precision: Int): String =
241     s"Frequency(${safeString(attribute)},${safeString(dtg)},$period,$precision)"
242 
243   /**
244     * String that will be parsed into a z3 count min sketch stat
245     *
246     * @param geom geometry attribute
247     * @param dtg date attribute
248     * @param period time period to split on
249     * @param precision precision of the z value - @see FrequencyZ3
250     * @return
251     */
252   def Z3Frequency(geom: String, dtg: String, period: TimePeriod, precision: Int): String =
253     s"Z3Frequency(${safeString(geom)},${safeString(dtg)},$period,$precision)"
254 
255   /**
256     * String that will be parsed to a binned histogram stat
257     *
258     * @param attribute attribute name to histogram
259     * @param bins the number of bins to create
260     * @param min min value for the histogram
261     * @param max max value for the histogram
262     * @tparam T class type of the histogram attribute
263     * @return
264     */
265   def Histogram[T](attribute: String, bins: Int, min: T, max: T)(implicit ct: ClassTag[T]): String = {
266     val stringify = stringifier(ct.runtimeClass)
267     s"Histogram(${safeString(attribute)},$bins,${safeString(stringify(min))},${safeString(stringify(max))})"
268   }
269 
270   /**
271     * String that will be parsed into a z3 range histogram stat
272     *
273     * @param geom geometry attribute
274     * @param dtg date attribute
275     * @param period time period to split on
276     * @param length number of the bins per week - @see RangeHistogramZ3
277     * @return
278     */
279   def Z3Histogram(geom: String, dtg: String, period: TimePeriod, length: Int): String =
280     s"Z3Histogram(${safeString(geom)},${safeString(dtg)},$period,$length)"
281 
282   /**
283     * String that will be parsed to a iterator stack counter
284     *
285     * @return
286     */
287   def IteratorStackCount(): String = "IteratorStackCount()"
288 
289   /**
290     * String that will be parsed to a sequence of stat
291     *
292     * @param stats input strings that will be parsed as individual stats
293     * @return
294     */
295   def SeqStat(stats: Seq[String]): String = stats.mkString(";")
296 
297   /**
298     * Groups results by attribute and runs stats for each group.
299     *
300     * @param attribute attribute to group stats by
301     * @param groupedStat stat to apply to grouped attributes
302     * @return
303     */
304   def GroupBy(attribute: String, groupedStat: String): String = s"GroupBy(${safeString(attribute)},$groupedStat)"
305 
306   /**
307     * String that will be parsed into a multi variate descriptive stat
308     *
309     * @param attributes attribute name to evaluate
310     * @return
311     */
312   def DescriptiveStats(attributes: Seq[String]): String = s"DescriptiveStats(${attributes.map(safeString).mkString(",")})"
313 
314   /**
315     * Combines a sequence of stats. This will not modify any of the inputs.
316     *
317     * @param stats stats to combine
318     * @return
319     */
320   def combine[T <: Stat](stats: Seq[T]): Option[T] = {
321     if (stats.length < 2) {
322       stats.headOption
323     } else {
324       // create a new stat so that we don't modify the existing ones
325       val summed = stats.head + stats.tail.head
326       stats.drop(2).foreach(summed += _)
327       Some(summed.asInstanceOf[T])
328     }
329   }
330 
331   // note: adds quotes around the string
332   private def safeString(s: String): String = s""""${StringEscapeUtils.escapeJava(s)}""""
333 
334   /**
335     * Converts a value to a string
336     *
337     * @param clas class of the input value
338     * @param json will the result be used in json? will quote appropriately if so
339     * @tparam T type of input
340     * @return
341     */
342   def stringifier[T](clas: Class[T], json: Boolean = false): Any => String = {
343     val toString: Any => String = if (classOf[Geometry].isAssignableFrom(clas)) {
344       v => WKTUtils.write(v.asInstanceOf[Geometry])
345     } else if (classOf[Date].isAssignableFrom(clas)) {
346       v => GeoToolsDateFormat.format(toInstant(v.asInstanceOf[Date]))
347     } else {
348       v => v.toString
349     }
350 
351     // add quotes to json strings if needed
352     if (json && (!classOf[Number].isAssignableFrom(clas) && clas != classOf[java.lang.Boolean])) {
353       v => if (v == null) { "null" } else { s""""${toString(v)}"""" }
354     } else {
355       v => if (v == null) { "null" } else { toString(v) }
356     }
357   }
358 
359   /**
360     * Converts a string back to a value
361     *
362     * @param clas class of the stringified value
363     * @tparam T type of the value class
364     * @return
365     */
366   def destringifier[T](clas: Class[T]): String => T = {
367     if (clas == classOf[String]) {
368       s => if (s == "null") { null.asInstanceOf[T] } else { s.asInstanceOf[T] }
369     } else if (clas == classOf[Integer]) {
370       s => if (s == "null") { null.asInstanceOf[T] } else { s.toInt.asInstanceOf[T] }
371     } else if (clas == classOf[jLong]) {
372       s => if (s == "null") { null.asInstanceOf[T] } else { s.toLong.asInstanceOf[T] }
373     } else if (clas == classOf[jFloat]) {
374       s => if (s == "null") { null.asInstanceOf[T] } else { s.toFloat.asInstanceOf[T] }
375     } else if (clas == classOf[jDouble]) {
376       s => if (s == "null") { null.asInstanceOf[T] } else { s.toDouble.asInstanceOf[T] }
377     } else if (clas == classOf[java.lang.Boolean]) {
378       s => if (s == "null") { null.asInstanceOf[T] } else { s.toBoolean.asInstanceOf[T] }
379     } else if (classOf[Geometry].isAssignableFrom(clas)) {
380       s => if (s == "null") { null.asInstanceOf[T] } else { WKTUtils.read(s).asInstanceOf[T] }
381     } else if (classOf[Date].isAssignableFrom(clas)) {
382       s => if (s == "null") { null.asInstanceOf[T] } else {
383         Date.from(LocalDateTime.parse(s, GeoToolsDateFormat).toInstant(ZoneOffset.UTC)).asInstanceOf[T]
384       }
385     } else {
386       throw new RuntimeException(s"Unexpected class binding for stat attribute: $clas")
387     }
388   }
389 
390   trait ImmutableStat extends Stat {
391     override def observe(sf: SimpleFeature): Unit = fail()
392     override def unobserve(sf: SimpleFeature): Unit = fail()
393     override def +=(other: S): Unit = fail()
394     override def clear(): Unit = fail()
395 
396     private def fail(): Unit = throw new RuntimeException("This stat is immutable")
397   }
398 }