Skip to main content
Ctrl
+
K
Site Navigation
Overview
Getting Started
User Guides
API Reference
Development
Migration Guides
4.0.0-preview2
GitHub
PyPI
Site Navigation
Overview
Getting Started
User Guides
API Reference
Development
Migration Guides
4.0.0-preview2
GitHub
PyPI
Section Navigation
Spark SQL
Core Classes
pyspark.sql.SparkSession
pyspark.sql.Catalog
pyspark.sql.DataFrame
pyspark.sql.Column
pyspark.sql.Observation
pyspark.sql.Row
pyspark.sql.GroupedData
pyspark.sql.PandasCogroupedOps
pyspark.sql.DataFrameNaFunctions
pyspark.sql.DataFrameStatFunctions
pyspark.sql.Window
pyspark.sql.DataFrameReader
pyspark.sql.DataFrameWriter
pyspark.sql.DataFrameWriterV2
pyspark.sql.UDFRegistration
pyspark.sql.UDTFRegistration
pyspark.sql.udf.UserDefinedFunction
pyspark.sql.udtf.UserDefinedTableFunction
pyspark.sql.datasource.DataSource
pyspark.sql.datasource.DataSourceReader
pyspark.sql.datasource.DataSourceStreamReader
pyspark.sql.datasource.DataSourceWriter
pyspark.sql.datasource.DataSourceRegistration
pyspark.sql.datasource.InputPartition
pyspark.sql.datasource.WriterCommitMessage
pyspark.sql.VariantVal
Spark Session
pyspark.sql.SparkSession.active
pyspark.sql.SparkSession.builder.appName
pyspark.sql.SparkSession.builder.config
pyspark.sql.SparkSession.builder.enableHiveSupport
pyspark.sql.SparkSession.builder.getOrCreate
pyspark.sql.SparkSession.builder.master
pyspark.sql.SparkSession.builder.remote
pyspark.sql.SparkSession.catalog
pyspark.sql.SparkSession.conf
pyspark.sql.SparkSession.createDataFrame
pyspark.sql.SparkSession.dataSource
pyspark.sql.SparkSession.getActiveSession
pyspark.sql.SparkSession.newSession
pyspark.sql.SparkSession.profile
pyspark.sql.SparkSession.range
pyspark.sql.SparkSession.read
pyspark.sql.SparkSession.readStream
pyspark.sql.SparkSession.sparkContext
pyspark.sql.SparkSession.sql
pyspark.sql.SparkSession.stop
pyspark.sql.SparkSession.streams
pyspark.sql.SparkSession.table
pyspark.sql.SparkSession.udf
pyspark.sql.SparkSession.udtf
pyspark.sql.SparkSession.version
pyspark.sql.is_remote
pyspark.sql.SparkSession.builder.create
pyspark.sql.SparkSession.addArtifact
pyspark.sql.SparkSession.addArtifacts
pyspark.sql.SparkSession.addTag
pyspark.sql.SparkSession.clearProgressHandlers
pyspark.sql.SparkSession.clearTags
pyspark.sql.SparkSession.client
pyspark.sql.SparkSession.copyFromLocalToFs
pyspark.sql.SparkSession.getTags
pyspark.sql.SparkSession.interruptAll
pyspark.sql.SparkSession.interruptOperation
pyspark.sql.SparkSession.interruptTag
pyspark.sql.SparkSession.registerProgressHandler
pyspark.sql.SparkSession.removeProgressHandler
pyspark.sql.SparkSession.removeTag
Configuration
pyspark.sql.conf.RuntimeConfig
Input/Output
pyspark.sql.DataFrameReader.csv
pyspark.sql.DataFrameReader.format
pyspark.sql.DataFrameReader.jdbc
pyspark.sql.DataFrameReader.json
pyspark.sql.DataFrameReader.load
pyspark.sql.DataFrameReader.option
pyspark.sql.DataFrameReader.options
pyspark.sql.DataFrameReader.orc
pyspark.sql.DataFrameReader.parquet
pyspark.sql.DataFrameReader.schema
pyspark.sql.DataFrameReader.table
pyspark.sql.DataFrameReader.text
pyspark.sql.DataFrameWriter.bucketBy
pyspark.sql.DataFrameWriter.csv
pyspark.sql.DataFrameWriter.format
pyspark.sql.DataFrameWriter.insertInto
pyspark.sql.DataFrameWriter.jdbc
pyspark.sql.DataFrameWriter.json
pyspark.sql.DataFrameWriter.mode
pyspark.sql.DataFrameWriter.option
pyspark.sql.DataFrameWriter.options
pyspark.sql.DataFrameWriter.orc
pyspark.sql.DataFrameWriter.parquet
pyspark.sql.DataFrameWriter.partitionBy
pyspark.sql.DataFrameWriter.save
pyspark.sql.DataFrameWriter.saveAsTable
pyspark.sql.DataFrameWriter.sortBy
pyspark.sql.DataFrameWriter.text
pyspark.sql.DataFrameWriterV2.using
pyspark.sql.DataFrameWriterV2.option
pyspark.sql.DataFrameWriterV2.options
pyspark.sql.DataFrameWriterV2.tableProperty
pyspark.sql.DataFrameWriterV2.partitionedBy
pyspark.sql.DataFrameWriterV2.create
pyspark.sql.DataFrameWriterV2.replace
pyspark.sql.DataFrameWriterV2.createOrReplace
pyspark.sql.DataFrameWriterV2.append
pyspark.sql.DataFrameWriterV2.overwrite
pyspark.sql.DataFrameWriterV2.overwritePartitions
pyspark.sql.MergeIntoWriter.whenMatched
pyspark.sql.MergeIntoWriter.whenNotMatched
pyspark.sql.MergeIntoWriter.whenNotMatchedBySource
pyspark.sql.MergeIntoWriter.withSchemaEvolution
pyspark.sql.MergeIntoWriter.merge
DataFrame
pyspark.sql.DataFrame.__getattr__
pyspark.sql.DataFrame.__getitem__
pyspark.sql.DataFrame.agg
pyspark.sql.DataFrame.alias
pyspark.sql.DataFrame.approxQuantile
pyspark.sql.DataFrame.cache
pyspark.sql.DataFrame.checkpoint
pyspark.sql.DataFrame.coalesce
pyspark.sql.DataFrame.colRegex
pyspark.sql.DataFrame.collect
pyspark.sql.DataFrame.columns
pyspark.sql.DataFrame.corr
pyspark.sql.DataFrame.count
pyspark.sql.DataFrame.cov
pyspark.sql.DataFrame.createGlobalTempView
pyspark.sql.DataFrame.createOrReplaceGlobalTempView
pyspark.sql.DataFrame.createOrReplaceTempView
pyspark.sql.DataFrame.createTempView
pyspark.sql.DataFrame.crossJoin
pyspark.sql.DataFrame.crosstab
pyspark.sql.DataFrame.cube
pyspark.sql.DataFrame.describe
pyspark.sql.DataFrame.distinct
pyspark.sql.DataFrame.drop
pyspark.sql.DataFrame.dropDuplicates
pyspark.sql.DataFrame.dropDuplicatesWithinWatermark
pyspark.sql.DataFrame.drop_duplicates
pyspark.sql.DataFrame.dropna
pyspark.sql.DataFrame.dtypes
pyspark.sql.DataFrame.exceptAll
pyspark.sql.DataFrame.executionInfo
pyspark.sql.DataFrame.explain
pyspark.sql.DataFrame.fillna
pyspark.sql.DataFrame.filter
pyspark.sql.DataFrame.first
pyspark.sql.DataFrame.foreach
pyspark.sql.DataFrame.foreachPartition
pyspark.sql.DataFrame.freqItems
pyspark.sql.DataFrame.groupBy
pyspark.sql.DataFrame.groupingSets
pyspark.sql.DataFrame.head
pyspark.sql.DataFrame.hint
pyspark.sql.DataFrame.inputFiles
pyspark.sql.DataFrame.intersect
pyspark.sql.DataFrame.intersectAll
pyspark.sql.DataFrame.isEmpty
pyspark.sql.DataFrame.isLocal
pyspark.sql.DataFrame.isStreaming
pyspark.sql.DataFrame.join
pyspark.sql.DataFrame.limit
pyspark.sql.DataFrame.localCheckpoint
pyspark.sql.DataFrame.mapInPandas
pyspark.sql.DataFrame.mapInArrow
pyspark.sql.DataFrame.melt
pyspark.sql.DataFrame.na
pyspark.sql.DataFrame.observe
pyspark.sql.DataFrame.offset
pyspark.sql.DataFrame.orderBy
pyspark.sql.DataFrame.persist
pyspark.sql.DataFrame.printSchema
pyspark.sql.DataFrame.randomSplit
pyspark.sql.DataFrame.rdd
pyspark.sql.DataFrame.registerTempTable
pyspark.sql.DataFrame.repartition
pyspark.sql.DataFrame.repartitionByRange
pyspark.sql.DataFrame.replace
pyspark.sql.DataFrame.rollup
pyspark.sql.DataFrame.sameSemantics
pyspark.sql.DataFrame.sample
pyspark.sql.DataFrame.sampleBy
pyspark.sql.DataFrame.schema
pyspark.sql.DataFrame.select
pyspark.sql.DataFrame.selectExpr
pyspark.sql.DataFrame.semanticHash
pyspark.sql.DataFrame.show
pyspark.sql.DataFrame.sort
pyspark.sql.DataFrame.sortWithinPartitions
pyspark.sql.DataFrame.sparkSession
pyspark.sql.DataFrame.stat
pyspark.sql.DataFrame.storageLevel
pyspark.sql.DataFrame.subtract
pyspark.sql.DataFrame.summary
pyspark.sql.DataFrame.tail
pyspark.sql.DataFrame.take
pyspark.sql.DataFrame.to
pyspark.sql.DataFrame.toArrow
pyspark.sql.DataFrame.toDF
pyspark.sql.DataFrame.toJSON
pyspark.sql.DataFrame.toLocalIterator
pyspark.sql.DataFrame.toPandas
pyspark.sql.DataFrame.transform
pyspark.sql.DataFrame.union
pyspark.sql.DataFrame.unionAll
pyspark.sql.DataFrame.unionByName
pyspark.sql.DataFrame.unpersist
pyspark.sql.DataFrame.unpivot
pyspark.sql.DataFrame.where
pyspark.sql.DataFrame.withColumn
pyspark.sql.DataFrame.withColumns
pyspark.sql.DataFrame.withColumnRenamed
pyspark.sql.DataFrame.withColumnsRenamed
pyspark.sql.DataFrame.withMetadata
pyspark.sql.DataFrame.withWatermark
pyspark.sql.DataFrame.write
pyspark.sql.DataFrame.writeStream
pyspark.sql.DataFrame.writeTo
pyspark.sql.DataFrame.mergeInto
pyspark.sql.DataFrame.pandas_api
pyspark.sql.DataFrameNaFunctions.drop
pyspark.sql.DataFrameNaFunctions.fill
pyspark.sql.DataFrameNaFunctions.replace
pyspark.sql.DataFrameStatFunctions.approxQuantile
pyspark.sql.DataFrameStatFunctions.corr
pyspark.sql.DataFrameStatFunctions.cov
pyspark.sql.DataFrameStatFunctions.crosstab
pyspark.sql.DataFrameStatFunctions.freqItems
pyspark.sql.DataFrameStatFunctions.sampleBy
Column
pyspark.sql.Column.__getattr__
pyspark.sql.Column.__getitem__
pyspark.sql.Column.alias
pyspark.sql.Column.asc
pyspark.sql.Column.asc_nulls_first
pyspark.sql.Column.asc_nulls_last
pyspark.sql.Column.astype
pyspark.sql.Column.between
pyspark.sql.Column.bitwiseAND
pyspark.sql.Column.bitwiseOR
pyspark.sql.Column.bitwiseXOR
pyspark.sql.Column.cast
pyspark.sql.Column.contains
pyspark.sql.Column.desc
pyspark.sql.Column.desc_nulls_first
pyspark.sql.Column.desc_nulls_last
pyspark.sql.Column.dropFields
pyspark.sql.Column.endswith
pyspark.sql.Column.eqNullSafe
pyspark.sql.Column.getField
pyspark.sql.Column.getItem
pyspark.sql.Column.ilike
pyspark.sql.Column.isNaN
pyspark.sql.Column.isNotNull
pyspark.sql.Column.isNull
pyspark.sql.Column.isin
pyspark.sql.Column.like
pyspark.sql.Column.name
pyspark.sql.Column.otherwise
pyspark.sql.Column.over
pyspark.sql.Column.rlike
pyspark.sql.Column.startswith
pyspark.sql.Column.substr
pyspark.sql.Column.try_cast
pyspark.sql.Column.when
pyspark.sql.Column.withField
Data Types
ArrayType
BinaryType
BooleanType
ByteType
DataType
DateType
DecimalType
DoubleType
FloatType
IntegerType
LongType
MapType
NullType
ShortType
StringType
CharType
VarcharType
StructField
StructType
VariantType
TimestampType
TimestampNTZType
DayTimeIntervalType
YearMonthIntervalType
CalendarIntervalType
Row
pyspark.sql.Row.asDict
Functions
pyspark.sql.functions.broadcast
pyspark.sql.functions.call_function
pyspark.sql.functions.col
pyspark.sql.functions.column
pyspark.sql.functions.lit
pyspark.sql.functions.expr
pyspark.sql.functions.coalesce
pyspark.sql.functions.ifnull
pyspark.sql.functions.nanvl
pyspark.sql.functions.nullif
pyspark.sql.functions.nullifzero
pyspark.sql.functions.nvl
pyspark.sql.functions.nvl2
pyspark.sql.functions.when
pyspark.sql.functions.zeroifnull
pyspark.sql.functions.equal_null
pyspark.sql.functions.ilike
pyspark.sql.functions.isnan
pyspark.sql.functions.isnotnull
pyspark.sql.functions.isnull
pyspark.sql.functions.like
pyspark.sql.functions.regexp
pyspark.sql.functions.regexp_like
pyspark.sql.functions.rlike
pyspark.sql.functions.asc
pyspark.sql.functions.asc_nulls_first
pyspark.sql.functions.asc_nulls_last
pyspark.sql.functions.desc
pyspark.sql.functions.desc_nulls_first
pyspark.sql.functions.desc_nulls_last
pyspark.sql.functions.abs
pyspark.sql.functions.acos
pyspark.sql.functions.acosh
pyspark.sql.functions.asin
pyspark.sql.functions.asinh
pyspark.sql.functions.atan
pyspark.sql.functions.atan2
pyspark.sql.functions.atanh
pyspark.sql.functions.bin
pyspark.sql.functions.bround
pyspark.sql.functions.cbrt
pyspark.sql.functions.ceil
pyspark.sql.functions.ceiling
pyspark.sql.functions.conv
pyspark.sql.functions.cos
pyspark.sql.functions.cosh
pyspark.sql.functions.cot
pyspark.sql.functions.csc
pyspark.sql.functions.degrees
pyspark.sql.functions.e
pyspark.sql.functions.exp
pyspark.sql.functions.expm1
pyspark.sql.functions.factorial
pyspark.sql.functions.floor
pyspark.sql.functions.greatest
pyspark.sql.functions.hex
pyspark.sql.functions.hypot
pyspark.sql.functions.least
pyspark.sql.functions.ln
pyspark.sql.functions.log
pyspark.sql.functions.log10
pyspark.sql.functions.log1p
pyspark.sql.functions.log2
pyspark.sql.functions.negate
pyspark.sql.functions.negative
pyspark.sql.functions.pi
pyspark.sql.functions.pmod
pyspark.sql.functions.positive
pyspark.sql.functions.pow
pyspark.sql.functions.power
pyspark.sql.functions.radians
pyspark.sql.functions.rand
pyspark.sql.functions.randn
pyspark.sql.functions.rint
pyspark.sql.functions.round
pyspark.sql.functions.sec
pyspark.sql.functions.sign
pyspark.sql.functions.signum
pyspark.sql.functions.sin
pyspark.sql.functions.sinh
pyspark.sql.functions.sqrt
pyspark.sql.functions.tan
pyspark.sql.functions.tanh
pyspark.sql.functions.try_add
pyspark.sql.functions.try_divide
pyspark.sql.functions.try_mod
pyspark.sql.functions.try_multiply
pyspark.sql.functions.try_subtract
pyspark.sql.functions.unhex
pyspark.sql.functions.width_bucket
pyspark.sql.functions.ascii
pyspark.sql.functions.base64
pyspark.sql.functions.bit_length
pyspark.sql.functions.btrim
pyspark.sql.functions.char
pyspark.sql.functions.char_length
pyspark.sql.functions.character_length
pyspark.sql.functions.collate
pyspark.sql.functions.collation
pyspark.sql.functions.concat_ws
pyspark.sql.functions.contains
pyspark.sql.functions.decode
pyspark.sql.functions.elt
pyspark.sql.functions.encode
pyspark.sql.functions.endswith
pyspark.sql.functions.find_in_set
pyspark.sql.functions.format_number
pyspark.sql.functions.format_string
pyspark.sql.functions.initcap
pyspark.sql.functions.instr
pyspark.sql.functions.lcase
pyspark.sql.functions.left
pyspark.sql.functions.length
pyspark.sql.functions.levenshtein
pyspark.sql.functions.locate
pyspark.sql.functions.lower
pyspark.sql.functions.lpad
pyspark.sql.functions.ltrim
pyspark.sql.functions.mask
pyspark.sql.functions.octet_length
pyspark.sql.functions.overlay
pyspark.sql.functions.position
pyspark.sql.functions.printf
pyspark.sql.functions.regexp_count
pyspark.sql.functions.regexp_extract
pyspark.sql.functions.regexp_extract_all
pyspark.sql.functions.regexp_instr
pyspark.sql.functions.regexp_replace
pyspark.sql.functions.regexp_substr
pyspark.sql.functions.repeat
pyspark.sql.functions.replace
pyspark.sql.functions.right
pyspark.sql.functions.rpad
pyspark.sql.functions.rtrim
pyspark.sql.functions.sentences
pyspark.sql.functions.soundex
pyspark.sql.functions.split
pyspark.sql.functions.split_part
pyspark.sql.functions.startswith
pyspark.sql.functions.substr
pyspark.sql.functions.substring
pyspark.sql.functions.substring_index
pyspark.sql.functions.to_binary
pyspark.sql.functions.to_char
pyspark.sql.functions.to_number
pyspark.sql.functions.to_varchar
pyspark.sql.functions.translate
pyspark.sql.functions.trim
pyspark.sql.functions.try_to_binary
pyspark.sql.functions.try_to_number
pyspark.sql.functions.ucase
pyspark.sql.functions.unbase64
pyspark.sql.functions.upper
pyspark.sql.functions.bit_count
pyspark.sql.functions.bit_get
pyspark.sql.functions.bitwise_not
pyspark.sql.functions.getbit
pyspark.sql.functions.shiftleft
pyspark.sql.functions.shiftright
pyspark.sql.functions.shiftrightunsigned
pyspark.sql.functions.add_months
pyspark.sql.functions.convert_timezone
pyspark.sql.functions.curdate
pyspark.sql.functions.current_date
pyspark.sql.functions.current_timestamp
pyspark.sql.functions.current_timezone
pyspark.sql.functions.date_add
pyspark.sql.functions.date_diff
pyspark.sql.functions.date_format
pyspark.sql.functions.date_from_unix_date
pyspark.sql.functions.date_part
pyspark.sql.functions.date_sub
pyspark.sql.functions.date_trunc
pyspark.sql.functions.dateadd
pyspark.sql.functions.datediff
pyspark.sql.functions.datepart
pyspark.sql.functions.day
pyspark.sql.functions.dayname
pyspark.sql.functions.dayofmonth
pyspark.sql.functions.dayofweek</