初始化项目

spark/dependencies.py

@@ -57,13 +57,9 @@ def md5(col):
     md5.update(str(col).encode('utf-8'))
     return md5.hexdigest()
 
-# 脱敏
-id = sys.argv[1]
-output_file=sys.argv[2]
-
 if __name__ == '__main__':
     # 拿到 列名 等
-    record = getRecordByIdResultT1(id)
+    record = getRecordByIdResultT1(sys.argv[1])
     col = json.loads(str(record)[2:-3]).get('col')
     sql = getColumns(str(record)[2:-3])
     print(sql)
@@ -73,7 +69,7 @@ if __name__ == '__main__':
 
     spark.udf.register('md5', md5, StringType())
     print('=======>', output_file)
-    df = spark.read.format('csv').option('inferSchema', 'true').load(output_file).toDF(*col)
+    df = spark.read.format('csv').option('inferSchema', 'true').load(sys.argv[2]).toDF(*col)
     df.count
     t_table = df.createTempView('kino')
     sinkDF = spark.sql(sql)