1import os 

2import subprocess 

3import tempfile 

4from contextlib import contextmanager, suppress 

5import shutil 

6import json 

7from distutils.dir_util import copy_tree 

8 

9@contextmanager 

10def _act_in_dir(new_dir): 

11 current_dir = os.getcwd() 

12 try: 

13 os.chdir(new_dir) 

14 yield 

15 finally: 

16 os.chdir(current_dir) 

17 

18def _delete_file_if_exist(filename): 

19 if os.path.isfile(filename): 

20 os.remove(filename) 

21 

22class Application: 

23 """Represent a spark application 

24 """ 

25 

26 def __init__(self, location): 

27 """ 

28 Parameters 

29 ---------- 

30 location: str 

31 The directory path for the source code location of the application. See examples/myapp for example. 

32 """ 

33 self.location = location 

34 with open(f"{self.location}/manifest.json", "r") as f: 

35 self.manifest = json.load(f) 

36 

37 @property 

38 def version(self): 

39 """The application version specified by the manifest file""" 

40 return self.manifest['version'] 

41 

42 def _create_venv(self): 

43 tmpdir = tempfile.mkdtemp() 

44 subprocess.check_call(['/usr/bin/python3', "-m", "venv", tmpdir]) 

45 subprocess.check_call([ 

46 os.path.join(tmpdir, "bin", "python"), 

47 "-m", "pip", 

48 "install","pip", "setuptools", "--upgrade" 

49 ]) 

50 subprocess.check_call([ 

51 os.path.join(tmpdir, "bin", "python"), 

52 "-m", "pip", "install","wheel" 

53 ]) 

54 return tmpdir 

55 

56 def _get_tmp_requirements(self, default_libs): 

57 with tempfile.NamedTemporaryFile(mode="w+t", delete=False) as f: 

58 for line in default_libs: 

59 print(line, file=f) 

60 app_req_filename = os.path.join(self.location, "requirements.txt") 

61 if os.path.isfile(app_req_filename): 

62 with open(app_req_filename, "rt") as arf: 

63 for line in arf: 

64 print(line, file=f) 

65 return f.name 

66 

67 

68 def build(self, destination, default_libs=[]): 

69 """ 

70 Build the application, it generates the necessary artifacts for the application including: 

71 app.zip -- the archive for the application code 

72 lib.zip -- the archive for the library 

73 main.py -- the main application entry 

74 manifest.json -- the application manifest file 

75 

76 Parameters 

77 ---------- 

78 location: str 

79 The directory where those generated artifacts is stored. 

80 

81 default_libs: list of str 

82 A list of libraries needed which is not specified in the requirements.txt. 

83 E.g. ["six==1.11.0"] 

84 """ 

85 

86 os.makedirs(destination, exist_ok=True) 

87 build_stage_dir = tempfile.mkdtemp() 

88 

89 venv_dir = self._create_venv() 

90 

91 lib_dir = os.path.join(build_stage_dir, "lib") 

92 os.mkdir(lib_dir) 

93 tmp_requirements = self._get_tmp_requirements(default_libs) 

94 subprocess.check_call([ 

95 os.path.join(venv_dir, "bin", "python"), 

96 "-m", "pip", 

97 "install", "-r", tmp_requirements, 

98 "-t", lib_dir 

99 ]) 

100 os.remove(tmp_requirements) 

101 

102 # generate archive for lib 

103 lib_filename = os.path.join(build_stage_dir, 'lib.zip') 

104 with _act_in_dir(lib_dir): 

105 subprocess.run(['zip', "-r", lib_filename, "."]) 

106 

107 # generate archive for app 

108 app_filename = os.path.join(build_stage_dir, 'app.zip') 

109 with _act_in_dir(self.location): 

110 subprocess.run(['zip', "-r", app_filename, "."]) 

111 

112 for filename in ["main.py", "manifest.json", "lib.zip", "app.zip"]: 

113 _delete_file_if_exist(os.path.join(destination, filename)) 

114 

115 shutil.copyfile(lib_filename, os.path.join(destination, 'lib.zip')) 

116 shutil.copyfile(app_filename, os.path.join(destination, 'app.zip')) 

117 shutil.copyfile( 

118 os.path.join(self.location, "manifest.json"), 

119 os.path.join(destination, 'manifest.json') 

120 ) 

121 shutil.copyfile( 

122 os.path.join(self.location, "main.py"), 

123 os.path.join(destination, 'main.py') 

124 ) 

125 shutil.rmtree(venv_dir) 

126 shutil.rmtree(build_stage_dir)