QuickStart

Provides classes and code snippets to simplify usage of Spatial APIs in Location Intelligence SDK For Big Data.

For more information regarding the Usage Guide or API Docs, follow the below links:

SparkSession.py
1
2
3
4
5
6
from pyspark.sql import SparkSession
session = SparkSession.builder() \
        .appName("Example") \
        .config("spark.jars", "/li_distribution/pyspark/sdk/lib/location-intelligence-bigdata-li-sdk-spark3_2.12-<version>.jar")
        .getOrCreate()
session.sparkContext.addPyFile('/li_distribution/pyspark/sdk/lib/location-intelligence-bigdata-li-sdk-pyspark-<version>.zip')

Using DownloadManager capable of downloading remote resources (files like TAB, SHAPE, etc.) to a node local path. Supports downloading from HDFS, S3, Google Storage or Local (default).

NOTE: For Downloading from S3 or Google Storage, you need to either have respective environment variables like AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY or provide configurations in spark session.

DownloadManager.py
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
from li.DownloadManagerBuilder import DownloadManagerBuilder
from li.LocalFilePassthroughDownloader import LocalFilePassthroughDownloader
from li.HDFSDownloader import HDFSDownloader
from li.S3Downloader import S3Downloader
from li.S3Downloader import GoogleDownloader
from li.HadoopConfiguration import HadoopConfiguration

# Create object of Download Manager
downloadManager = (DownloadManagerBuilder("./downloads")
    .addDownloader(S3Downloader(HadoopConfiguration().getHadoopConfiguration()).getDownloader())
    .addDownloader(GoogleDownloader(HadoopConfiguration().getHadoopConfiguration()).getDownloader())
    .addDownloader(HDFSDownloader(HadoopConfiguration().getHadoopConfiguration()).getDownloader())
    .addDownloader(LocalFilePassthroughDownloader().getDownloader())
    .build())

Following is the PointInPolygon Operation:

PointInPolygon.py
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
from li.SpatialAPI import SpatialAPI

pointInPolygonDF = SpatialAPI.pointInPolygon(inputDF = inputDF,
            tableFileType = tabFileType,
            tableFilePath = tableFilePath,
            tableFileName = tableFileName,
            libraries = libraries,
            longitude = longitude, latitude = latitude,
            includeEmptySearchResults = true,
            outputFields = outputFields,
            downloadManager = downloadManager)
# Defaults:
# downloadManager = None, libraries = None, includeEmptySearchResults = True

Following is the SearchNearest Operation:

SearchNearest.py
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
from li.SpatialAPI import SpatialAPI

searchNearestDF = SpatialAPI.searchNearest(inputDF = fabricDF,
              tableFileType = tableFileType,
              tableFilePath = tableFilePath,
              tableFileName = tableFileName,
              libraries = libraries,
              maxCandidates = maxCandidates,
              distanceValue = distanceValue,
              distanceUnit = distanceUnit,
              distanceColumnName = distanceColumnName,
              geometryStringType = geometryStringType,
              geometryColumnName = geometryColumnName,
              includeEmptySearchResults = includeEmptySearchResults,
              outputFields = outputFields,
              downloadManager = downloadManager)
# Defaults:
# distanceColumnName = "distance", downloadManager = None, libraries = None, includeEmptySearchResults = True, maxCandidates = 1000

Following is the JoinByDistance Operation:

JoinByDistance.py
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
from li.SpatialAPI import SpatialAPI
from li.DistanceJoinOption import DistanceJoinOption
from li.LimitMethods import LimitMethods

joinedDF = SpatialAPI.joinByDistance(df1 = df1,
                 df2 = df2,
                 df1Longitude = longitude1,
                 df1Latitude = latitude1,
                 df2Longitude = longitude2,
                 df2Latitude = latitude2,
                 searchRadius = searchRadius,
                 distanceUnit = distanceUnit,
                 geoHashPrecision = geoHashPrecision,
                 options = {DistanceJoinOption.DistanceColumnName: distanceColumnName, DistanceJoinOption.LimitMatches: limit, DistanceJoinOption.LimitMethod: LimitMethods.RowNumber})

# Defaults:
# geoHashPrecision = 7, options = None

Following is the GenerateHexagon Operation:

GenerateHexagon.py
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
from li.SpatialAPI import SpatialAPI
from li.DistanceJoinOption import DistanceJoinOption
from li.LimitMethods import LimitMethods

hexGenDF = SpatialAPI.generateHexagon(
                 sparkSession = session,
                 minLongitude = minLongitude,
                 minLatitude = minLatitude,
                 maxLongitude = maxLongitude,
                 maxLatitude = maxLatitude,
                 hexLevel = hexLevel,
                 containerLevel = containerLevel,
                 numOfPartitions = numOfPartitions,
                 maximumNumOfRowsPerPartition = maxNumberOfRows)

# Defaults:
# hexLevel = 1, containerLevel = 1, numOfPartitions = 1, maximumNumOfRowsPerPartition = 1

Following is the example for registering the SQL Functions as UDFs:

LI_SQL.py
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
# Importing the Required Classes
from li.SQLRegistrator import SQLRegistrator

# Register the LI UDFs
SQLRegistrator.registerAll()

# Create the TemporaryTable from input dataframe
inputDF.createOrReplaceTempView("inputTable")

# Use the UDF in a SQL query
pointGeometry = spark.sql("SELECT ST_Point(X, Y, 'epsg:4326') as point_geom, * from inputTable")
wktGeometry = spark.sql("SELECT ST_GeomFromWKT(WKT) as geom, * from inputTable")

# Currently Available SQL Functions
# ST_Point, ST_GeomFromWKT, ST_GeomFromWKB, ST_GeomFromKML, ST_GeomFromGeoJSON,
# ST_ToGeoJSON, ST_ToKML, ST_ToWKB, ST_ToWKT, ST_Buffer, ST_Union, ST_Transform,
# ST_Intersection, ST_ConvexHull, ST_Within, ST_Disjoint, ST_Intersects,
# ST_IsNullGeom, ST_Overlaps, ST_GeoHash, ST_GeoHashBoundary, ST_HexHash, ST_HexHashBoundary,
# ST_SquareHash, ST_SquareHashBoundary, ST_X, ST_XMax, ST_XMin, ST_Y, ST_YMax, ST_YMin,
# ST_Area, ST_Distance, ST_Length, ST_Perimeter