1 /***********************************************************************
2  * Copyright (c) 2013-2025 General Atomics Integrated Intelligence, Inc.
3  * All rights reserved. This program and the accompanying materials
4  * are made available under the terms of the Apache License, Version 2.0
5  * which accompanies this distribution and is available at
6  * https://www.apache.org/licenses/LICENSE-2.0
7  ***********************************************************************/
8 
9 package org.locationtech.geomesa.convert.json
10 
11 import com.google.gson._
12 import com.google.gson.stream.{JsonReader, JsonToken}
13 import com.jayway.jsonpath.spi.json.GsonJsonProvider
14 import com.jayway.jsonpath.{Configuration, JsonPath, PathNotFoundException}
15 import com.typesafe.config.Config
16 import org.geotools.api.feature.simple.SimpleFeatureType
17 import org.locationtech.geomesa.convert._
18 import org.locationtech.geomesa.convert2.AbstractConverter.BasicOptions
19 import org.locationtech.geomesa.convert2.transforms.Expression
20 import org.locationtech.geomesa.convert2.{AbstractConverter, ConverterConfig, ConverterName, Field}
21 import org.locationtech.geomesa.utils.collection.CloseableIterator
22 
23 import java.io._
24 import java.nio.charset.Charset
25 
26 class JsonConverter(sft: SimpleFeatureType, config: JsonConverter.JsonConfig, fields: Seq[JsonConverter.JsonField], options: BasicOptions)
27     extends AbstractConverter[JsonElement, JsonConverter.JsonConfig, JsonConverter.JsonField, BasicOptions](sft, config, fields, options) {
28 
29   import scala.collection.JavaConverters._
30 
31   private val featurePath = config.featurePath.map(JsonPath.compile(_))
32 
33   override protected def parse(is: InputStream, ec: EvaluationContext): CloseableIterator[JsonElement] =
34     new JsonConverter.JsonIterator(is, options.encoding, ec)
35 
36   override protected def values(parsed: CloseableIterator[JsonElement],
37                                 ec: EvaluationContext): CloseableIterator[Array[Any]] = {
38     val array = Array.ofDim[Any](2)
39     featurePath match {
40       case None =>
41         parsed.map { element =>
42           array(0) = element
43           array
44         }
45       case Some(path) =>
46         parsed.flatMap { element =>
47           array(1) = element
48           path.read[JsonArray](element, JsonConverter.JsonConfiguration).iterator.asScala.map { e =>
49             array(0) = e
50             array
51           }
52         }
53     }
54   }
55 }
56 
57 object JsonConverter extends GeoJsonParsing {
58 
59   private [json] val JsonConfiguration =
60     Configuration.builder()
61         .jsonProvider(new GsonJsonProvider)
62         .options(com.jayway.jsonpath.Option.DEFAULT_PATH_LEAF_TO_NULL)
63         .build()
64 
65   private val LineRegex = """JsonReader at line (\d+)""".r
66 
67   case class JsonConfig(
68       `type`: String,
69       converterName: Option[String],
70       featurePath: Option[String],
71       idField: Option[Expression],
72       caches: Map[String, Config],
73       userData: Map[String, Expression]
74     ) extends ConverterConfig with ConverterName
75 
76   sealed trait JsonField extends Field
77 
78   case class DerivedField(name: String, transforms: Option[Expression]) extends JsonField {
79     override val fieldArg: Option[Array[AnyRef] => AnyRef] = None
80   }
81 
82   abstract class TypedJsonField(val jsonType: String) extends JsonField {
83 
84     def path: String
85     def pathIsRoot: Boolean
86 
87     private val i = if (pathIsRoot) { 1 } else { 0 }
88     private val jsonPath = JsonPath.compile(path)
89 
90     protected def unwrap(elem: JsonElement): AnyRef
91 
92     override val fieldArg: Option[Array[AnyRef] => AnyRef] = Some(values)
93 
94     private def values(args: Array[AnyRef]): AnyRef = {
95       val e = try { jsonPath.read[JsonElement](args(i), JsonConfiguration) } catch {
96         case _: PathNotFoundException => JsonNull.INSTANCE
97       }
98       if (e.isJsonNull) { null } else { unwrap(e) }
99     }
100   }
101 
102   case class StringJsonField(name: String, path: String, pathIsRoot: Boolean, transforms: Option[Expression])
103       extends TypedJsonField("string") {
104     override def unwrap(elem: JsonElement): AnyRef = elem.getAsString
105   }
106 
107   case class FloatJsonField(name: String, path: String, pathIsRoot: Boolean, transforms: Option[Expression])
108       extends TypedJsonField("float") {
109     override def unwrap(elem: JsonElement): AnyRef = Float.box(elem.getAsFloat)
110   }
111 
112   case class DoubleJsonField(name: String, path: String, pathIsRoot: Boolean, transforms: Option[Expression])
113       extends TypedJsonField("double") {
114     override def unwrap(elem: JsonElement): AnyRef = Double.box(elem.getAsDouble)
115   }
116 
117   case class IntJsonField(name: String, path: String, pathIsRoot: Boolean, transforms: Option[Expression])
118       extends TypedJsonField("int") {
119     override def unwrap(elem: JsonElement): AnyRef = Int.box(elem.getAsInt)
120   }
121 
122   case class BooleanJsonField(name: String, path: String, pathIsRoot: Boolean, transforms: Option[Expression])
123       extends TypedJsonField("boolean") {
124     override def unwrap(elem: JsonElement): AnyRef = Boolean.box(elem.getAsBoolean)
125   }
126 
127   case class LongJsonField(name: String, path: String, pathIsRoot: Boolean, transforms: Option[Expression])
128       extends TypedJsonField("long") {
129     override def unwrap(elem: JsonElement): AnyRef = Long.box(elem.getAsBigInteger.longValue())
130   }
131 
132   case class GeometryJsonField(name: String, path: String, pathIsRoot: Boolean, transforms: Option[Expression])
133       extends TypedJsonField("geometry") {
134     override def unwrap(elem: JsonElement): AnyRef = parseGeometry(elem)
135   }
136 
137   case class ArrayJsonField(name: String, path: String, pathIsRoot: Boolean, transforms: Option[Expression])
138       extends TypedJsonField("array") {
139     override def unwrap(elem: JsonElement): AnyRef = elem.getAsJsonArray
140   }
141 
142   case class ObjectJsonField(name: String, path: String, pathIsRoot: Boolean, transforms: Option[Expression])
143       extends TypedJsonField("object") {
144     override def unwrap(elem: JsonElement): AnyRef = elem.getAsJsonObject
145   }
146 
147   /**
148     * Parses an input stream into json elements
149     *
150     * @param is input
151     * @param encoding encoding
152     * @param ec context
153     */
154   class JsonIterator private [json] (is: InputStream, encoding: Charset, ec: EvaluationContext)
155       extends CloseableIterator[JsonElement] {
156 
157     private val reader = new JsonReader(new InputStreamReader(is, encoding))
158     reader.setLenient(true)
159 
160     override def hasNext: Boolean = reader.peek() != JsonToken.END_DOCUMENT
161     override def next(): JsonElement = {
162       val res = JsonParser.parseReader(reader)
163       // extract the line number, only accessible from reader.toString
164       LineRegex.findFirstMatchIn(reader.toString).foreach(m => ec.line = m.group(1).toLong)
165       res
166     }
167 
168     override def close(): Unit = reader.close()
169   }
170 }