package api
This section provides code snippets to simplify usage of Geo Addressing SDK for Big Data.
Geocode Operation Example
import org.apache.spark.sql.functions._ import com.pb.downloadmanager.api._ import com.pb.downloadmanager.api.downloaders._ import com.pb.downloadmanager.api.downloaders.hadoop._ import com.precisely.addressing.v1.model._ import com.precisely.bigdata.addressing.spark.api._ import com.precisely.addressing.v1._ // DownloadManager copies all the remote resources (HDFS, S3, Google Storage, etc.) to the // provided local path (e.g. ./downloads) of every worker node. // This happens in runtime during the first call. Subsequent calls skips downloading process. val downloadManager = new DownloadManagerBuilder("./downloads") .addDownloader(new S3Downloader(spark.sparkContext.hadoopConfiguration)) .addDownloader(new GoogleDownloader(spark.sparkContext.hadoopConfiguration)) .addDownloader(new HDFSDownloader(spark.sparkContext.hadoopConfiguration)) .addDownloader(new LocalFilePassthroughDownloader()) .build() // Remove the fields which are not required. val outputFields = Seq( "location.feature.geometry.coordinates.x as LON", "location.feature.geometry.coordinates.y as LAT", "address.formattedAddress as fullAddress", "address.city.longName as locality", "address.country.isoAlpha2Code as countryshortname", "address.country.name as country", "address.placeName as placename", "address.admin1.shortName as administrativearealevel1", "address.admin2.longName as administrativearealevel2", "address.suburb as suburb", "address.borough as borough", "address.postalCode as postalcode", "address.postalCodeExt as postalcodesuffix", "customFields['PB_KEY'] as sourceplaceid", "address.addressNumber as AddressNumber", "address.street as Street", "address.unit as Unit", "address.unitType as UnitType", "address.formattedStreetAddress as StreetAddress", "address.formattedLocationAddress as LocationAddress", "address.neighborhood as neighborhood", "address.floor as floor", "score", "explanation.source['label'] as label", "customFields['PRECISION_CODE'] as PrecisionCode", "customFields['MATCH_TYPE'] as MatchType", "customFields['LOC_CODE'] as LocationCode", "customFields['MATCH_CODE'] as MatchCode", "customFields['CBSA_NAME'] as CBSAName", "customFields['PREF_CITY'] as PREFCITY", "customFields['PB_KEY'] as PBKEY", "customFields['DATATYPE_NAME'] as DataTypeName" ) // You can also use paths from HDFS or Databricks Volumes // (e.g. hdfs:///addressingDistribution/resources or /Volumes/precisely/default/geo-addressing/addressingDistribution/resources/) // The reference data is extracted at runtime in every worker node's extractionLocation path. val geocodeUdf = new AddressingBuilder() .withResourcesLocation("s3a:///addressingDistribution/resources") .withDownloadManager(downloadManager) .withDataLocations("s3a://com.precisely.data/path_to_spd1","s3a://path_to_spd2") .withExtractionLocation("./extracted") .udfBuilder() .withPreferences(new PreferencesBuilder().withReturnAllInfo(true).build()) .withOutputFields(outputFields: _*) .withErrorField("error") .withResultAsJSON("jsonOutput") .forGeocode() val geocodeDF = input.withColumn("result", geocodeUdf(map( lit("addressLines[0]"), col("address"), lit("country"), col("country") ))).select("*", "result.*").drop(colName = "result")
NOTE: If addressing.yaml is the present at the worker node, dataLocations and
extractionLocation is preferred from addressing.yaml over code.
In such cases where addressing.yaml is present, you should only provide resourcesLocation.
Custom Executor Example for Multipass Geocoding:
import com.precisely.bigdata.addressing.spark.api._ import org.apache.spark.sql.functions._ // Refer to Geocode Operation Example for Full Code. // Use .forCustomExecutor() API val customUdf = new AddressingBuilder() ... .forCustomExecutor(new AddressingExecutor { override def execute(input: RequestInput, preferences: Option[Preferences], addressing: Addressing): Response = { // First Pass val response = addressing.geocode(input.requestAddress(), preferences.orNull) // Validate the results. if (results.get(0).getCustomFields.get("PRECISION_CODE").toUpperCase().startsWith("S8")) return response // Second Pass // Convert MultiLine Input Address to SingleLine val singleLineAddress: RequestAddress = new RequestAddress() val filterAddress: String = Seq(inputMultiLineAddress.getStreet, inputMultiLineAddress.getCity, inputMultiLineAddress.getAdmin1, inputMultiLineAddress.getPostalCode ).filter(isNotNull).mkString(" ") singleLineAddress.setAddressLines(java.util.Collections.singletonList(filterAddress: String)) singleLineAddress.setCountry(inputMultiLineAddress.getCountry) // Geocode with SingleLineAddress and CustomPreferences val singleLineResponse: Response = addressing.geocode(singleLineAddress, new PreferencesBuilder().withReturnAllInfo(true).build()) // Adding a Custom Field singleLineResponse.getResults.get(0).getCustomFields.put("TOTAL_MATCHES", singleLineResponse.getResults.size().toString) return singleLineResponse } override def execute(lookupType: LookupType, preferences: Option[Preferences], addressing: Addressing, keyValues: KeyValue*): Response = ??? override def execute(x: Double, y: Double, country: String, preferences: Option[Preferences], addressing: Addressing): Response = ??? }) val geocodeDF = input.withColumn("result", customUdf(map( lit("addressLines[0]"), col("address"), lit("country"), col("country") ))).select("*", "result.*").drop(colName = "result")
Verify Operation Example:
import com.precisely.bigdata.addressing.spark.api.AddressingBuilder import org.apache.spark.sql.functions._ // Refer to Geocode Operation Example for Full Code. // Use .forVerify() API val verifyUdf = new AddressingBuilder() ... .forVerify() val verifyDf = input.withColumn("result", verifyUdf(map( lit("addressLines[0]"), col("address"), lit("country"), col("country") ))).select("*", "result.*").drop(colName = "result")
Reverse Geocode Operation Example:
import com.precisely.bigdata.addressing.spark.api.AddressingBuilder import org.apache.spark.sql.functions._ // Refer to Geocode Operation Example for Full Code. // Use .forReverseGeocode() API val reverseGeocodeUdf = new AddressingBuilder() ... .forReverseGeocode() val reverseGeocodeDf = input.withColumn("result", reverseGeocodeUdf( col("x"), col("y"), col("country") )).select("*", "result.*").drop(colName = "result")
Lookup Operation Example:
import com.precisely.bigdata.addressing.spark.api.AddressingBuilder import org.apache.spark.sql.functions._ // Refer to Geocode Operation Example for Full Code. // Use .forLookup() API val lookupUdf = new AddressingBuilder() ... .forLookup() val lookUpUdf = input.withColumn("result", lookupUdf( col("keyType"), col("key"), col("country") )).select("*", "result.*").drop(colName = "result")
Geocode Operation with SQL Queries:
import com.precisely.bigdata.addressing.spark.api.AddressingBuilder import org.apache.spark.sql.functions._ // Refer to Geocode Operation Example for Full Code. // Use .forGeocode(<udf-name>) API val geocodeUdf = new AddressingBuilder() ... .forGeocode("PreciselyGeocode") inputDF.createOrReplaceTempView("inputTable") val geocodeDF = spark.sql("select *, PreciselyGeocode(map('addressLines[0]', address, 'country', country)) as result from inputTable")
Linear Supertypes
Type Members
- class AddressingBuilder extends AnyRef
This class allows you to either build an AddressingProvider for use in your code, or to branch off into a builder for a UDF that executes an addressing operation.
- trait AddressingExecutor extends Serializable
An implementation of a custom Addressing API operation.
- trait AddressingProvider extends Serializable
A serializable provider for the Addressing API.
A serializable provider for the Addressing API. This instance can be used in spark tasks for direct Addressing API usage since it is serializable.
- class MapFunctionBuilder extends AnyRef
This class allows you to build a UDF that executes an addressing operation.
This class allows you to build a UDF that executes an addressing operation. You can obtain an instance of this builder by starting with an AddressingBuilder.
- trait RequestInput extends Serializable
Provides access to the input for a given addressing operation, as well as the option to build a RequestAddress.
- class UDFBuilder extends AnyRef
This class allows you to build a UDF that executes an addressing operation.
This class allows you to build a UDF that executes an addressing operation. You can obtain an instance of this builder by starting with an AddressingBuilder.