{"id":24402,"library":"pyspark-extension","title":"pyspark-extension","description":"A library providing useful extensions to Apache Spark, including DataFrame diff, column transformation utilities, Parquet metadata reading, Spark Connect support, and dependency installation helpers. Current version 2.15.0.4.1, supports Spark 3.2+, 4.0; release cadence is irregular with multiple releases per year.","status":"active","version":"2.15.0.4.1","language":"python","source_language":"en","source_url":"https://github.com/G-Research/spark-extension","tags":["apache-spark","pyspark","data-diff","dataframe","spark-extension"],"install":[{"cmd":"pip install pyspark-extension","lang":"bash","label":"Standard install"},{"cmd":"pip install pyspark-extension[scala]","lang":"bash","label":"With Scala/Java support"}],"dependencies":[{"reason":"Core dependency for Spark functionality","package":"pyspark","optional":false},{"reason":"Used for Java bridge","package":"py4j","optional":false}],"imports":[{"wrong":"from spark_extension import diff","symbol":"diff","correct":"from gresearch.spark.diff import diff"},{"symbol":"ParquetMetadata","correct":"from gresearch.spark.parquet import ParquetMetadata"},{"symbol":"ColumnTransformer","correct":"from gresearch.spark.column import ColumnTransformer"}],"quickstart":{"code":"from pyspark.sql import SparkSession\nfrom spark_extension import diff\n\nspark = SparkSession.builder.appName(\"test\").getOrCreate()\ndf1 = spark.createDataFrame([(1, \"a\"), (2, \"b\")], [\"id\", \"val\"])\ndf2 = spark.createDataFrame([(1, \"a\"), (3, \"c\")], [\"id\", \"val\"])\nresult = diff(df1, df2)\nresult.show()","lang":"python","description":"Basic usage: diff two DataFrames"},"warnings":[{"fix":"Use Spark 3.2+ or install pyspark-extension==2.14.2","message":"Removed support for Spark 3.0 and 3.1 starting from version 2.15.0. Upgrade to Spark 3.2+ or pin pyspark-extension <2.15.0.","severity":"breaking","affected_versions":">=2.15.0"},{"fix":"Only use documented public API; check release notes for symbols that became private.","message":"All undocumented unintended public API parts were made private in version 2.15.0. Any use of internal symbols (e.g., _internal methods) will break.","severity":"breaking","affected_versions":">=2.15.0"},{"fix":"Install with [scala] extra or configure spark.jars","message":"The Java/Scala jar is required for some features (e.g., diff on DataFrames with complex types). Install with pip install pyspark-extension[scala] or manually add the jar to Spark session.","severity":"gotcha","affected_versions":"all"},{"fix":"Review column name quoting in any custom SQL using column names from this library.","message":"Backticks handling: In version 2.14.0, columns with special characters are quoted with backticks; columns with only alphanumerics and underscores are no longer quoted. This may break comparisons relying on quoted column names in SQL expressions.","severity":"deprecated","affected_versions":">=2.14.0"}],"env_vars":null,"search_vec":"'2.15.0.4.1':30 '3.2':33 '4.0':34 'apach':10,45 'apache-spark':44 'cadenc':36 'column':15 'connect':22 'current':28 'data':49 'data-diff':48 'datafram':13,51 'depend':25 'diff':14,50 'extens':3,8,54 'helper':27 'includ':12 'instal':26 'irregular':38 'librari':5 'metadata':19 'multipl':40 'parquet':18 'per':42 'provid':6 'pyspark':2,47 'pyspark-extens':1 'read':20 'releas':35,41 'spark':11,21,32,46,53 'spark-extens':52 'support':23,31 'transform':16 'use':7 'util':17 'version':29 'year':43","created_at":"2026-05-01T08:12:53.973584+00:00","updated_at":"2026-05-01T08:12:53.973584+00:00","problems":{"verify_error":"Traceback (most recent call last):\n  File \"<string>\", line 1, in <module>\n  File \"/tmp/tmp3riln5wo/venv/lib/python3.12/site-packages/gresearch/spark/__init__.py\", line 26, in <module>\n    from py4j.java_gateway import JVMView, JavaObject\nModuleNotFoundError: No module named 'py4j'"},"ecosystem":"pypi","meta_description":null,"install_score":null,"quickstart_score":null,"quickstart_tag":null,"pypi_latest":"2.15.0.4.1","cli_name":"","cli_version":null,"type":"library","homepage":null,"github":"https://github.com/G-Research/spark-extension","docs":null,"changelog":null,"pypi":"https://pypi.org/project/pyspark-extension/","npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"categories":["data","workflow"],"base_url":null,"auth_type":null,"provenance":{"verified_status":"import_fail","verified_at":"2026-07-03","last_verified":"2026-07-03","next_check":"2026-07-10","install_tag":null}}