Skip to content

W
wanjia
Commit 7e34eec0 authored by kinomin's avatar kinomin
Browse files
Options

初始化项目

parent 879c50e3
Pipeline #290 failed with stages
      Show whitespace changes
      Inline Side-by-side
      Showing with 2 additions and 6 deletions
      spark/dependencies.py
      View file @ 7e34eec0
      ...@@ -57,13 +57,9 @@ def md5(col): ...@@ -57,13 +57,9 @@ def md5(col):
      md5.update(str(col).encode('utf-8')) md5.update(str(col).encode('utf-8'))
      return md5.hexdigest() return md5.hexdigest()
      # 脱敏
      id = sys.argv[1]
      output_file=sys.argv[2]
      if __name__ == '__main__': if __name__ == '__main__':
      # 拿到 列名 等 # 拿到 列名 等
      record = getRecordByIdResultT1(id) record = getRecordByIdResultT1(sys.argv[1])
      col = json.loads(str(record)[2:-3]).get('col') col = json.loads(str(record)[2:-3]).get('col')
      sql = getColumns(str(record)[2:-3]) sql = getColumns(str(record)[2:-3])
      print(sql) print(sql)
      ...@@ -73,7 +69,7 @@ if __name__ == '__main__': ...@@ -73,7 +69,7 @@ if __name__ == '__main__':
      spark.udf.register('md5', md5, StringType()) spark.udf.register('md5', md5, StringType())
      print('=======>', output_file) print('=======>', output_file)
      df = spark.read.format('csv').option('inferSchema', 'true').load(output_file).toDF(*col) df = spark.read.format('csv').option('inferSchema', 'true').load(sys.argv[2]).toDF(*col)
      df.count df.count
      t_table = df.createTempView('kino') t_table = df.createTempView('kino')
      sinkDF = spark.sql(sql) sinkDF = spark.sql(sql)
      ......
      Markdown is supported
      0% or
      You are about to add 0 people to the discussion. Proceed with caution.
      Finish editing this message first!
      Please register or to comment