{"body":"spark_use_local_conda = False\n\n# the bundled local conda environment.\n# this is required if the spark workers require\n# modules that are not installed on the default wmf conda base environment\nif spark_use_local_conda:\n    spark_env_file = 'spark_venv.tar.gz'\n    if os.path.isfile(spark_env_file):\n        print(f\"the conda environment is packed already in file {spark_env_file}. if you installed new dependencies, delete {spark_env_file} first or use a different file name\")\n    else:\n        !conda pack --force --ignore-editable-packages -o {spark_env_file}\n\n# set the python on the worker machines\nif spark_use_local_conda:\n    # upload the bundled local conda environment\n    os.environ['PYSPARK_SUBMIT_ARGS'] = f'--archives {spark_env_file}#venv pyspark-shell'\n    os.environ['PYSPARK_PYTHON'] = 'venv/bin/python'\nelse:\n    # use the base wmf conda environment\n    os.environ['PYSPARK_PYTHON'] = '/usr/lib/anaconda-wmf/bin/python'\n","name":"","extension":"txt","url":"https://www.irccloud.com/pastebin/dQVrXPSq","modified":1613010455,"id":"dQVrXPSq","size":938,"lines":20,"own_paste":false,"theme":"","date":1613010455}