<jupyter_start><jupyter_text><jupyter_code>!pip install pyspark
from pyspark.sql import session
import pyspark.sql

from pyspark.sql import SparkSession
spark = SparkSession.builder.appName("firstApp").getOrCreate()
spark
from google.colab import files

files.upload()
df = spark.read.csv("GOOG.csv",header=True,inferSchema = True)
df
df.columns
df.printSchema()
df.head()
for row in df.head(10):
  print(row)
df.describe().show()
from pyspark.sql.functions import format_number
description = df.describe()
description.select(description["summary"],format_number(description["Open"].cast("float"),2).alias("Open"),
                   format_number(description["High"].cast("float"),2).alias("High"),
                   format_number(description["Low"].cast("float"),2).alias("Low"),
                   format_number(description["Close"].cast("float"),2).alias("Close"),
                   description["Volume"].cast("int").alias("Volume")
                   ).show()
new_df = df.withColumn("Open-VolumeRatio", df["Open"]/df["Volume"])
new_df.show()
new_df.select("Open-VolumeRatio").show()
df.orderBy(df["High"].desc()).show()
from pyspark.sql.functions import mean,max,min

df_mix_mx_mean = df.select(mean("Open").alias("MeanOpen"),max("Open").alias("MaxOpen"),min("Open").alias("MinOpen"))
df_mix_mx_mean.show()
df.select(max("Open").alias("MaxOpen")).show()

df_Vollessthan2M = df.filter("Volume < 2000000")
from pyspark.sql.functions import count
df_Vollessthan2M.select(count("Volume")).show()
from pyspark.sql.functions import corr
df. select(corr("Open","Volume")).show()
from pyspark.sql.functions import year
df_with_year = df.withColumn("Year",year(df["Date"]))
df_with_year.show()
from pyspark.sql import SparkSession

app = SparkSession.builder.appName("SQL").getOrCreate()
df.createOrReplaceTempView("stock")

app.sql("select * from stock").show()

app.sql("SELECT * FROM stock WHERE High> 3000 ").show()<jupyter_output>+----------+-----------+-----------+-----------+-----------+-----------+-------+
|      Date|       Open|       High|        Low|      Close|  Adj Close| Volume|
+----------+-----------+-----------+-----------+-----------+-----------+-------+
|2021-11-05|2987.070068|3011.415039|2973.040039|2984.820068|2984.820068|1019700|
|2021-11-08|     3000.0|3020.689941|2982.399902|2987.030029|2987.030029| 919400|
|2021-11-09|2994.919922|3007.570068|2950.139893|2984.969971|2984.969971| 843800|
|2021-11-15|     3000.0|3009.540039|2973.050049| 2987.76001| 2987.76001| 812400|
|2021-11-18|2982.919922|3032.199951|2979.969971|3014.179932|3014.179932|1332900|
|2021-11-19|     3020.0|     3037.0|    2997.75|2999.050049|2999.050049| 988300|
|2021-11-22|3002.834961|3014.889893|2940.110107|2941.570068|2941.570068|1230400|
+----------+-----------+-----------+-----------+-----------+-----------+-------+
StarcoderdataJupyterStructuredCleanDedup