logo
  • Getting Started
  • User Guide
  • API Reference
  • Development
  • Migration Guide
  • Spark SQL
    • pyspark.sql.SparkSession
    • pyspark.sql.Catalog
    • pyspark.sql.DataFrame
    • pyspark.sql.Column
    • pyspark.sql.Row
    • pyspark.sql.GroupedData
    • pyspark.sql.PandasCogroupedOps
    • pyspark.sql.DataFrameNaFunctions
    • pyspark.sql.DataFrameStatFunctions
    • pyspark.sql.Window
    • pyspark.sql.SparkSession.builder.appName
    • pyspark.sql.SparkSession.builder.config
    • pyspark.sql.SparkSession.builder.enableHiveSupport
    • pyspark.sql.SparkSession.builder.getOrCreate
    • pyspark.sql.SparkSession.builder.master
    • pyspark.sql.SparkSession.catalog
    • pyspark.sql.SparkSession.conf
    • pyspark.sql.SparkSession.createDataFrame
    • pyspark.sql.SparkSession.getActiveSession
    • pyspark.sql.SparkSession.newSession
    • pyspark.sql.SparkSession.range
    • pyspark.sql.SparkSession.read
    • pyspark.sql.SparkSession.readStream
    • pyspark.sql.SparkSession.sparkContext
    • pyspark.sql.SparkSession.sql
    • pyspark.sql.SparkSession.stop
    • pyspark.sql.SparkSession.streams
    • pyspark.sql.SparkSession.table
    • pyspark.sql.SparkSession.udf
    • pyspark.sql.SparkSession.version
    • pyspark.sql.conf.RuntimeConfig
    • pyspark.sql.DataFrameReader.csv
    • pyspark.sql.DataFrameReader.format
    • pyspark.sql.DataFrameReader.jdbc
    • pyspark.sql.DataFrameReader.json
    • pyspark.sql.DataFrameReader.load
    • pyspark.sql.DataFrameReader.option
    • pyspark.sql.DataFrameReader.options
    • pyspark.sql.DataFrameReader.orc
    • pyspark.sql.DataFrameReader.parquet
    • pyspark.sql.DataFrameReader.schema
    • pyspark.sql.DataFrameReader.table
    • pyspark.sql.DataFrameWriter.bucketBy
    • pyspark.sql.DataFrameWriter.csv
    • pyspark.sql.DataFrameWriter.format
    • pyspark.sql.DataFrameWriter.insertInto
    • pyspark.sql.DataFrameWriter.jdbc
    • pyspark.sql.DataFrameWriter.json
    • pyspark.sql.DataFrameWriter.mode
    • pyspark.sql.DataFrameWriter.option
    • pyspark.sql.DataFrameWriter.options
    • pyspark.sql.DataFrameWriter.orc
    • pyspark.sql.DataFrameWriter.parquet
    • pyspark.sql.DataFrameWriter.partitionBy
    • pyspark.sql.DataFrameWriter.save
    • pyspark.sql.DataFrameWriter.saveAsTable
    • pyspark.sql.DataFrameWriter.sortBy
    • pyspark.sql.DataFrameWriter.text
    • pyspark.sql.DataFrame.agg
    • pyspark.sql.DataFrame.alias
    • pyspark.sql.DataFrame.approxQuantile
    • pyspark.sql.DataFrame.cache
    • pyspark.sql.DataFrame.checkpoint
    • pyspark.sql.DataFrame.coalesce
    • pyspark.sql.DataFrame.colRegex
    • pyspark.sql.DataFrame.collect
    • pyspark.sql.DataFrame.columns
    • pyspark.sql.DataFrame.corr
    • pyspark.sql.DataFrame.count
    • pyspark.sql.DataFrame.cov
    • pyspark.sql.DataFrame.createGlobalTempView
    • pyspark.sql.DataFrame.createOrReplaceGlobalTempView
    • pyspark.sql.DataFrame.createOrReplaceTempView
    • pyspark.sql.DataFrame.createTempView
    • pyspark.sql.DataFrame.crossJoin
    • pyspark.sql.DataFrame.crosstab
    • pyspark.sql.DataFrame.cube
    • pyspark.sql.DataFrame.describe
    • pyspark.sql.DataFrame.distinct
    • pyspark.sql.DataFrame.drop
    • pyspark.sql.DataFrame.dropDuplicates
    • pyspark.sql.DataFrame.drop_duplicates
    • pyspark.sql.DataFrame.dropna
    • pyspark.sql.DataFrame.dtypes
    • pyspark.sql.DataFrame.exceptAll
    • pyspark.sql.DataFrame.explain
    • pyspark.sql.DataFrame.fillna
    • pyspark.sql.DataFrame.filter
    • pyspark.sql.DataFrame.first
    • pyspark.sql.DataFrame.foreach
    • pyspark.sql.DataFrame.foreachPartition
    • pyspark.sql.DataFrame.freqItems
    • pyspark.sql.DataFrame.groupBy
    • pyspark.sql.DataFrame.head
    • pyspark.sql.DataFrame.hint
    • pyspark.sql.DataFrame.inputFiles
    • pyspark.sql.DataFrame.intersect
    • pyspark.sql.DataFrame.intersectAll
    • pyspark.sql.DataFrame.isLocal
    • pyspark.sql.DataFrame.isStreaming
    • pyspark.sql.DataFrame.join
    • pyspark.sql.DataFrame.limit
    • pyspark.sql.DataFrame.localCheckpoint
    • pyspark.sql.DataFrame.mapInPandas
    • pyspark.sql.DataFrame.na
    • pyspark.sql.DataFrame.orderBy
    • pyspark.sql.DataFrame.persist
    • pyspark.sql.DataFrame.printSchema
    • pyspark.sql.DataFrame.randomSplit
    • pyspark.sql.DataFrame.rdd
    • pyspark.sql.DataFrame.registerTempTable
    • pyspark.sql.DataFrame.repartition
    • pyspark.sql.DataFrame.repartitionByRange
    • pyspark.sql.DataFrame.replace
    • pyspark.sql.DataFrame.rollup
    • pyspark.sql.DataFrame.sameSemantics
    • pyspark.sql.DataFrame.sample
    • pyspark.sql.DataFrame.sampleBy
    • pyspark.sql.DataFrame.schema
    • pyspark.sql.DataFrame.select
    • pyspark.sql.DataFrame.selectExpr
    • pyspark.sql.DataFrame.semanticHash
    • pyspark.sql.DataFrame.show
    • pyspark.sql.DataFrame.sort
    • pyspark.sql.DataFrame.sortWithinPartitions
    • pyspark.sql.DataFrame.stat
    • pyspark.sql.DataFrame.storageLevel
    • pyspark.sql.DataFrame.subtract
    • pyspark.sql.DataFrame.summary
    • pyspark.sql.DataFrame.tail
    • pyspark.sql.DataFrame.take
    • pyspark.sql.DataFrame.toDF
    • pyspark.sql.DataFrame.toJSON
    • pyspark.sql.DataFrame.toLocalIterator
    • pyspark.sql.DataFrame.toPandas
    • pyspark.sql.DataFrame.transform
    • pyspark.sql.DataFrame.union
    • pyspark.sql.DataFrame.unionAll
    • pyspark.sql.DataFrame.unionByName
    • pyspark.sql.DataFrame.unpersist
    • pyspark.sql.DataFrame.where
    • pyspark.sql.DataFrame.withColumn
    • pyspark.sql.DataFrame.withColumnRenamed
    • pyspark.sql.DataFrame.withWatermark
    • pyspark.sql.DataFrame.write
    • pyspark.sql.DataFrame.writeStream
    • pyspark.sql.DataFrame.writeTo
    • pyspark.sql.DataFrame.to_pandas_on_spark
    • pyspark.sql.DataFrameNaFunctions.drop
    • pyspark.sql.DataFrameNaFunctions.fill
    • pyspark.sql.DataFrameNaFunctions.replace
    • pyspark.sql.DataFrameStatFunctions.approxQuantile
    • pyspark.sql.DataFrameStatFunctions.corr
    • pyspark.sql.DataFrameStatFunctions.cov
    • pyspark.sql.DataFrameStatFunctions.crosstab
    • pyspark.sql.DataFrameStatFunctions.freqItems
    • pyspark.sql.DataFrameStatFunctions.sampleBy
    • pyspark.sql.Column.alias
    • pyspark.sql.Column.asc
    • pyspark.sql.Column.asc_nulls_first
    • pyspark.sql.Column.asc_nulls_last
    • pyspark.sql.Column.astype
    • pyspark.sql.Column.between
    • pyspark.sql.Column.bitwiseAND
    • pyspark.sql.Column.bitwiseOR
    • pyspark.sql.Column.bitwiseXOR
    • pyspark.sql.Column.cast
    • pyspark.sql.Column.contains
    • pyspark.sql.Column.desc
    • pyspark.sql.Column.desc_nulls_first
    • pyspark.sql.Column.desc_nulls_last
    • pyspark.sql.Column.dropFields
    • pyspark.sql.Column.endswith
    • pyspark.sql.Column.eqNullSafe
    • pyspark.sql.Column.getField
    • pyspark.sql.Column.getItem
    • pyspark.sql.Column.isNotNull
    • pyspark.sql.Column.isNull
    • pyspark.sql.Column.isin
    • pyspark.sql.Column.like
    • pyspark.sql.Column.name
    • pyspark.sql.Column.otherwise
    • pyspark.sql.Column.over
    • pyspark.sql.Column.rlike
    • pyspark.sql.Column.startswith
    • pyspark.sql.Column.substr
    • pyspark.sql.Column.when
    • pyspark.sql.Column.withField
    • ArrayType
    • BinaryType
    • BooleanType
    • ByteType
    • DataType
    • DateType
    • DecimalType
    • DoubleType
    • FloatType
    • IntegerType
    • LongType
    • MapType
    • NullType
    • ShortType
    • StringType
    • StructField
    • StructType
    • TimestampType
    • pyspark.sql.Row.asDict
    • pyspark.sql.functions.abs
    • pyspark.sql.functions.acos
    • pyspark.sql.functions.acosh
    • pyspark.sql.functions.add_months
    • pyspark.sql.functions.aggregate
    • pyspark.sql.functions.approxCountDistinct
    • pyspark.sql.functions.approx_count_distinct
    • pyspark.sql.functions.array
    • pyspark.sql.functions.array_contains
    • pyspark.sql.functions.array_distinct
    • pyspark.sql.functions.array_except
    • pyspark.sql.functions.array_intersect
    • pyspark.sql.functions.array_join
    • pyspark.sql.functions.array_max
    • pyspark.sql.functions.array_min
    • pyspark.sql.functions.array_position
    • pyspark.sql.functions.array_remove
    • pyspark.sql.functions.array_repeat
    • pyspark.sql.functions.array_sort
    • pyspark.sql.functions.array_union
    • pyspark.sql.functions.arrays_overlap
    • pyspark.sql.functions.arrays_zip
    • pyspark.sql.functions.asc
    • pyspark.sql.functions.asc_nulls_first
    • pyspark.sql.functions.asc_nulls_last
    • pyspark.sql.functions.ascii
    • pyspark.sql.functions.asin
    • pyspark.sql.functions.asinh
    • pyspark.sql.functions.assert_true
    • pyspark.sql.functions.atan
    • pyspark.sql.functions.atanh
    • pyspark.sql.functions.atan2
    • pyspark.sql.functions.avg
    • pyspark.sql.functions.base64
    • pyspark.sql.functions.bin
    • pyspark.sql.functions.bitwise_not
    • pyspark.sql.functions.bitwiseNOT
    • pyspark.sql.functions.broadcast
    • pyspark.sql.functions.bround
    • pyspark.sql.functions.bucket
    • pyspark.sql.functions.cbrt
    • pyspark.sql.functions.ceil
    • pyspark.sql.functions.coalesce
    • pyspark.sql.functions.col
    • pyspark.sql.functions.collect_list
    • pyspark.sql.functions.collect_set
    • pyspark.sql.functions.column
    • pyspark.sql.functions.concat
    • pyspark.sql.functions.concat_ws
    • pyspark.sql.functions.conv
    • pyspark.sql.functions.corr
    • pyspark.sql.functions.cos
    • pyspark.sql.functions.cosh
    • pyspark.sql.functions.count
    • pyspark.sql.functions.count_distinct
    • pyspark.sql.functions.countDistinct
    • pyspark.sql.functions.covar_pop
    • pyspark.sql.functions.covar_samp
    • pyspark.sql.functions.crc32
    • pyspark.sql.functions.create_map
    • pyspark.sql.functions.cume_dist
    • pyspark.sql.functions.current_date
    • pyspark.sql.functions.current_timestamp
    • pyspark.sql.functions.date_add
    • pyspark.sql.functions.date_format
    • pyspark.sql.functions.date_sub
    • pyspark.sql.functions.date_trunc
    • pyspark.sql.functions.datediff
    • pyspark.sql.functions.dayofmonth
    • pyspark.sql.functions.dayofweek
    • pyspark.sql.functions.dayofyear
    • pyspark.sql.functions.days
    • pyspark.sql.functions.decode
    • pyspark.sql.functions.degrees
    • pyspark.sql.functions.dense_rank
    • pyspark.sql.functions.desc
    • pyspark.sql.functions.desc_nulls_first
    • pyspark.sql.functions.desc_nulls_last
    • pyspark.sql.functions.element_at
    • pyspark.sql.functions.encode
    • pyspark.sql.functions.exists
    • pyspark.sql.functions.exp
    • pyspark.sql.functions.explode
    • pyspark.sql.functions.explode_outer
    • pyspark.sql.functions.expm1
    • pyspark.sql.functions.expr
    • pyspark.sql.functions.factorial
    • pyspark.sql.functions.filter
    • pyspark.sql.functions.first
    • pyspark.sql.functions.flatten
    • pyspark.sql.functions.floor
    • pyspark.sql.functions.forall
    • pyspark.sql.functions.format_number
    • pyspark.sql.functions.format_string
    • pyspark.sql.functions.from_csv
    • pyspark.sql.functions.from_json
    • pyspark.sql.functions.from_unixtime
    • pyspark.sql.functions.from_utc_timestamp
    • pyspark.sql.functions.get_json_object
    • pyspark.sql.functions.greatest
    • pyspark.sql.functions.grouping
    • pyspark.sql.functions.grouping_id
    • pyspark.sql.functions.hash
    • pyspark.sql.functions.hex
    • pyspark.sql.functions.hour
    • pyspark.sql.functions.hours
    • pyspark.sql.functions.hypot
    • pyspark.sql.functions.initcap
    • pyspark.sql.functions.input_file_name
    • pyspark.sql.functions.instr
    • pyspark.sql.functions.isnan
    • pyspark.sql.functions.isnull
    • pyspark.sql.functions.json_tuple
    • pyspark.sql.functions.kurtosis
    • pyspark.sql.functions.lag
    • pyspark.sql.functions.last
    • pyspark.sql.functions.last_day
    • pyspark.sql.functions.lead
    • pyspark.sql.functions.least
    • pyspark.sql.functions.length
    • pyspark.sql.functions.levenshtein
    • pyspark.sql.functions.lit
    • pyspark.sql.functions.locate
    • pyspark.sql.functions.log
    • pyspark.sql.functions.log10
    • pyspark.sql.functions.log1p
    • pyspark.sql.functions.log2
    • pyspark.sql.functions.lower
    • pyspark.sql.functions.lpad
    • pyspark.sql.functions.ltrim
    • pyspark.sql.functions.map_concat
    • pyspark.sql.functions.map_entries
    • pyspark.sql.functions.map_filter
    • pyspark.sql.functions.map_from_arrays
    • pyspark.sql.functions.map_from_entries
    • pyspark.sql.functions.map_keys
    • pyspark.sql.functions.map_values
    • pyspark.sql.functions.map_zip_with
    • pyspark.sql.functions.max
    • pyspark.sql.functions.md5
    • pyspark.sql.functions.mean
    • pyspark.sql.functions.min
    • pyspark.sql.functions.minute
    • pyspark.sql.functions.monotonically_increasing_id
    • pyspark.sql.functions.month
    • pyspark.sql.functions.months
    • pyspark.sql.functions.months_between
    • pyspark.sql.functions.nanvl
    • pyspark.sql.functions.next_day
    • pyspark.sql.functions.nth_value
    • pyspark.sql.functions.ntile
    • pyspark.sql.functions.overlay
    • pyspark.sql.functions.pandas_udf
    • pyspark.sql.functions.percent_rank
    • pyspark.sql.functions.percentile_approx
    • pyspark.sql.functions.posexplode
    • pyspark.sql.functions.posexplode_outer
    • pyspark.sql.functions.pow
    • pyspark.sql.functions.product
    • pyspark.sql.functions.quarter
    • pyspark.sql.functions.radians
    • pyspark.sql.functions.raise_error
    • pyspark.sql.functions.rand
    • pyspark.sql.functions.randn
    • pyspark.sql.functions.rank
    • pyspark.sql.functions.regexp_extract
    • pyspark.sql.functions.regexp_replace
    • pyspark.sql.functions.repeat
    • pyspark.sql.functions.reverse
    • pyspark.sql.functions.rint
    • pyspark.sql.functions.round
    • pyspark.sql.functions.row_number
    • pyspark.sql.functions.rpad
    • pyspark.sql.functions.rtrim
    • pyspark.sql.functions.schema_of_csv
    • pyspark.sql.functions.schema_of_json
    • pyspark.sql.functions.second
    • pyspark.sql.functions.sentences
    • pyspark.sql.functions.sequence
    • pyspark.sql.functions.session_window
    • pyspark.sql.functions.sha1
    • pyspark.sql.functions.sha2
    • pyspark.sql.functions.shiftleft
    • pyspark.sql.functions.shiftright
    • pyspark.sql.functions.shiftrightunsigned
    • pyspark.sql.functions.shuffle
    • pyspark.sql.functions.signum
    • pyspark.sql.functions.sin
    • pyspark.sql.functions.sinh
    • pyspark.sql.functions.size
    • pyspark.sql.functions.skewness
    • pyspark.sql.functions.slice
    • pyspark.sql.functions.sort_array
    • pyspark.sql.functions.soundex
    • pyspark.sql.functions.spark_partition_id
    • pyspark.sql.functions.split
    • pyspark.sql.functions.sqrt
    • pyspark.sql.functions.stddev
    • pyspark.sql.functions.stddev_pop
    • pyspark.sql.functions.stddev_samp
    • pyspark.sql.functions.struct
    • pyspark.sql.functions.substring
    • pyspark.sql.functions.substring_index
    • pyspark.sql.functions.sum
    • pyspark.sql.functions.sum_distinct
    • pyspark.sql.functions.sumDistinct
    • pyspark.sql.functions.tan
    • pyspark.sql.functions.tanh
    • pyspark.sql.functions.timestamp_seconds
    • pyspark.sql.functions.toDegrees
    • pyspark.sql.functions.toRadians
    • pyspark.sql.functions.to_csv
    • pyspark.sql.functions.to_date
    • pyspark.sql.functions.to_json
    • pyspark.sql.functions.to_timestamp
    • pyspark.sql.functions.to_utc_timestamp
    • pyspark.sql.functions.transform
    • pyspark.sql.functions.transform_keys
    • pyspark.sql.functions.transform_values
    • pyspark.sql.functions.translate
    • pyspark.sql.functions.trim
    • pyspark.sql.functions.trunc
    • pyspark.sql.functions.udf
    • pyspark.sql.functions.unbase64
    • pyspark.sql.functions.unhex
    • pyspark.sql.functions.unix_timestamp
    • pyspark.sql.functions.upper
    • pyspark.sql.functions.var_pop
    • pyspark.sql.functions.var_samp
    • pyspark.sql.functions.variance
    • pyspark.sql.functions.weekofyear
    • pyspark.sql.functions.when
    • pyspark.sql.functions.window
    • pyspark.sql.functions.xxhash64
    • pyspark.sql.functions.year
    • pyspark.sql.functions.years
    • pyspark.sql.functions.zip_with
    • pyspark.sql.avro.functions.from_avro
    • pyspark.sql.avro.functions.to_avro
    • pyspark.sql.Window.currentRow
    • pyspark.sql.Window.orderBy
    • pyspark.sql.Window.partitionBy
    • pyspark.sql.Window.rangeBetween
    • pyspark.sql.Window.rowsBetween
    • pyspark.sql.Window.unboundedFollowing
    • pyspark.sql.Window.unboundedPreceding
    • pyspark.sql.WindowSpec.orderBy
    • pyspark.sql.WindowSpec.partitionBy
    • pyspark.sql.WindowSpec.rangeBetween
    • pyspark.sql.WindowSpec.rowsBetween
    • pyspark.sql.GroupedData.agg
    • pyspark.sql.GroupedData.apply
    • pyspark.sql.GroupedData.applyInPandas
    • pyspark.sql.GroupedData.avg
    • pyspark.sql.GroupedData.cogroup
    • pyspark.sql.GroupedData.count
    • pyspark.sql.GroupedData.max
    • pyspark.sql.GroupedData.mean
    • pyspark.sql.GroupedData.min
    • pyspark.sql.GroupedData.pivot
    • pyspark.sql.GroupedData.sum
    • pyspark.sql.PandasCogroupedOps.applyInPandas
    • pyspark.sql.Catalog.cacheTable
    • pyspark.sql.Catalog.clearCache
    • pyspark.sql.Catalog.createExternalTable
    • p