ncusi
/
secret_life_of_CVEs
connected to https://github.com/ncusi/secret_life_of_CVEs.git


  
1

	
2

	
3

	
4

	
5

	
6

	
7

	
8

	
9

	
10

	
11

	
12

	
13

	
14

	
15

	
16

	
17

	
18

	
19

	
20

	
21

	
22

	
23

	
24

	
25

	
26

	
27

	
28

	
29

	
30

	
31

	
32

	
33

	
34

	
35

	
36

	
37

	
38

	
39

	
40

	
41

	
42

	
43

	
44

	
45

	
46

	
47

	
48

	
49

	
50

	
51

	
52

	
53

	
54

	
55

	
56

	
57

	
58

	
59

	
60

	
61

	
62

	
63

	
64

	
65

	
66

	
67

	
68

	
69

	
70

	
71

	
72

	
73

	
74

	
75

	
76

	
77

	
78

	
79

	
80

	
81

	
82

	
83

	
84

	
85

	
86

	
87

	
88

	
89

	
90

	
91

	
92

	
93

	
94

	
95

	
96

	
97

	
98

	
99

	
100

	
101

	
102

	
103

	
104

	
105

	
106

	
107

	
108

	
109

	
110

	
111

	
112

	
113

	
114

	
115

	
116

	
117

	
118

	
119

	
120

	
121

	
122

	
123

	
124

	
125

	
126

	
127

	
128

	
129

	
130

	
131

	
132

	
133

	
134

	
135

	
136

	
137

	
138

	
139

	
140

	
141

	
142

	
143

	
144

	
145

	
146

	
147

	
148

	
149

	
150

	
151

	
152

	
153

	
154

	
155

	
156

	
157

	
158

	
159

	
160

	
161

	
162

	
163

	
164

	
165

	
166

	
167

	
168

	
169

	
170

	
171

	
172

	
173

	
174

	
175

	
176

	
177

	
178

	
179

	
180

	
181

	
182

	
183

	
184

	
185

	
186

	
187

	
188

	
189

	
190

	
191

	
192

	
193

	
194

	
195

	
196

	
197

	
198

	
199

	
200

	
201

	
202

	
203

	
204

	
205

	
206

	
207

	
208

	
209

	
210

	
211

	
212

	
213

	
214

	
215

	
216

	
217

	
218

	
219

	
220

	
221

	
222

	
223

	
224

	
225

	
226

	
227

	
228

	
229

	
230

	
231

	
232

	
233

	
234

	
235

	
236

	
237

	
238

	
239

	
240

	
241

	
242

	
243

	
244

	
245

	
246

	
247

	
248

	
249

	
250

	
251

	
252

	
253

	
254

	
255

	
256

	
257

	
258

	
            schema: '2.0'
stages:
  retrieve_cve_dates:
    cmd: python3 dependency_search/retrieve_cve_dates.py data/cve_df_filename data/published_cve_df_filename
    deps:
    - path: data/cve_df_filename
      md5: 2edd1805e47670ff381b6e81f5b03463
      size: 60583695
    - path: dependency_search/retrieve_cve_dates.py
      md5: 1334463f333601c8a0b86ec9b69b6bbf
      size: 2064
    outs:
    - path: data/published_cve_df_filename
      md5: cac0d75d900b0904b5f79c7e35a2beb9
      size: 775181
  retrieve_cve_info:
    cmd: python3 cve_information/retrieve_cve_info.py data/published_cve_df_filename
      data/unique_cve_info-cvss_cwe_etc.parquet
    deps:
    - path: cve_information/retrieve_cve_info.py
      md5: 555e621cac7a1b3da6d461848175a5aa
      size: 3407
    - path: data/published_cve_df_filename
      md5: cac0d75d900b0904b5f79c7e35a2beb9
      size: 775181
    outs:
    - path: data/unique_cve_info-cvss_cwe_etc.parquet
      md5: 11d3d0ba71838bf742de7d766c736aa1
      size: 869228
  retrieve_project_info:
    cmd: python3 projects_stats/retrieve_metadata_from_WoC_mongodb.py data/cve_df_filename
      data/unique_project_info.parquet
    deps:
    - path: data/cve_df_filename
      md5: 2edd1805e47670ff381b6e81f5b03463
      size: 60583695
    - path: projects_stats/retrieve_metadata_from_WoC_mongodb.py
      md5: af3fe986de04dc659cfccf408d74ac17
      size: 9578
    outs:
    - path: data/unique_project_info.parquet
      md5: ab9166e42442236878e119a9c8fcfe49
      size: 2664107
  merge_cve_dates:
    cmd: python3 dependency_search/merge_cve_df_with_published_cve_df.py data/cve_df_filename
      data/published_cve_df_filename data/commits_with_published_cve_df
    deps:
    - path: data/cve_df_filename
      md5: 2edd1805e47670ff381b6e81f5b03463
      size: 60583695
    - path: data/published_cve_df_filename
      md5: cac0d75d900b0904b5f79c7e35a2beb9
      size: 775181
    - path: dependency_search/merge_cve_df_with_published_cve_df.py
      md5: 92cde17947d19098c1d1a5a33f0afc56
      size: 1237
    outs:
    - path: data/commits_with_published_cve_df
      md5: 3e8506d1bba834b6edf8a5e7f3528037
      size: 76845293
  add_cvss_rankings:
    cmd: python3 cve_information/add_cvss_ranking.py data/unique_cve_info-cvss_cwe_etc.parquet
      data/unique_cve_info-cvss_cwe_ranking_etc.parquet
    deps:
    - path: cve_information/add_cvss_ranking.py
      md5: 2a6c2aaae82a0f4b77fc14f23fe43b50
      size: 4346
    - path: data/unique_cve_info-cvss_cwe_etc.parquet
      md5: 11d3d0ba71838bf742de7d766c736aa1
      size: 869228
    outs:
    - path: data/unique_cve_info-cvss_cwe_ranking_etc.parquet
      md5: 2fd6b6671e1c0b8c8c93049d58b294d4
      size: 917236
  extension_to_language:
    cmd: python3 dependency_search/prepare_extension_to_language_dict.py data/extension_to_language.json
    deps:
    - path: dependency_search/prepare_extension_to_language_dict.py
      md5: 355116c3fa764839007208e247ba8740
      size: 2032
    - path: https://raw.githubusercontent.com/github/linguist/master/lib/linguist/languages.yml
      checksum: '"63d3d5d3b8efd4fd6b3bbe70490e6a60f3a47a934af098966717a242b04fb81b"'
      size: 136180
    outs:
    - path: data/extension_to_language.json
      md5: 1d02992b8f88b6502b110c192f85aba6
      size: 19299
  languages_for_cve:
    cmd: python3 dependency_search/find_programming_language_for_cve.py data/commits_with_published_cve_df
      data/extension_to_language.json data/combined_df
    deps:
    - path: data/commits_with_published_cve_df
      md5: 3e8506d1bba834b6edf8a5e7f3528037
      size: 76845293
    - path: data/extension_to_language.json
      md5: 1d02992b8f88b6502b110c192f85aba6
      size: 19299
    - path: dependency_search/find_programming_language_for_cve.py
      md5: 260ae5fab08772353acd86546540ac51
      size: 3263
    outs:
    - path: data/combined_df
      md5: ee44b4e1b17386cdf5e43bce507ba2c1
      size: 80110439
  language_to_class:
    cmd: python3 dependency_search/prepare_language_to_class_dict.py data/language_to_class.json
    deps:
    - path: dependency_search/prepare_language_to_class_dict.py
      md5: 6a743e89e8bcec27515faa1bbc7fc0d8
      size: 3711
    outs:
    - path: data/language_to_class.json
      md5: 3de6771a0070cc9551d6b40ad849e2c8
      size: 4163
  clean_combined_df:
    cmd: python3 dependency_search/clean_data_before_cve_lifespan_calculation.py data/combined_df
      data/dep_df data/language_to_class.json data/cleaned_cve_df
    deps:
    - path: data/combined_df
      md5: ee44b4e1b17386cdf5e43bce507ba2c1
      size: 80110439
    - path: data/dep_df
      md5: 462062bf333cd25670e3e65a9abb980f
      size: 30932536
    - path: data/language_to_class.json
      md5: 3de6771a0070cc9551d6b40ad849e2c8
      size: 4163
    - path: dependency_search/clean_data_before_cve_lifespan_calculation.py
      md5: f9beb4940f98c45379e736eb1a44c9df
      size: 4659
    outs:
    - path: data/cleaned_cve_df
      md5: 8cf88416f88c4c17c1e2c6abfe394a6d
      size: 46143556
  compute_cve_lifespan:
    cmd: python3 dependency_search/calculate_cve_lifespan_per_project.py data/cleaned_cve_df
      data/language_to_class.json data/cve_lifespan_language_df data/cve_lifespan_df
    deps:
    - path: data/cleaned_cve_df
      md5: 8cf88416f88c4c17c1e2c6abfe394a6d
      size: 46143556
    - path: data/language_to_class.json
      md5: 3de6771a0070cc9551d6b40ad849e2c8
      size: 4163
    - path: dependency_search/calculate_cve_lifespan_per_project.py
      md5: a5c3e41eeee69c7b6cb5bf54f5bd2845
      size: 16190
    outs:
    - path: data/cve_lifespan_df
      md5: 487bede33e432111583544c9109d4333
      size: 3042007
    - path: data/cve_lifespan_language_df
      md5: 833e440fa4bbc62c4f320067054e0870
      size: 5139223
  merge_cve_info_into_lifespan:
    cmd: python3 dependency_search/merge_cve_info_and_cve_lifespan.py data/unique_cve_info-cvss_cwe_ranking_etc.parquet
      data/cve_lifespan_language_df data/cve_lifespan_df data/cve_lifespan_and_cve_info_df
      data/cve_lifespan_language_and_cve_info_df
    deps:
    - path: data/cve_lifespan_df
      md5: 487bede33e432111583544c9109d4333
      size: 3042007
    - path: data/cve_lifespan_language_df
      md5: 833e440fa4bbc62c4f320067054e0870
      size: 5139223
    - path: data/unique_cve_info-cvss_cwe_ranking_etc.parquet
      md5: 2fd6b6671e1c0b8c8c93049d58b294d4
      size: 917236
    - path: dependency_search/merge_cve_info_and_cve_lifespan.py
      md5: d8fbc2694d8bd46735090bd1080d3b4d
      size: 1609
    outs:
    - path: data/cve_lifespan_and_cve_info_df
      md5: 7dfa44782d07e90c4a46bc8cdd64b355
      size: 5860513
    - path: data/cve_lifespan_language_and_cve_info_df
      md5: 0107e90bacf138012923ee2d3b4be23c
      size: 3738093
  evaluate-cve_surv:
    cmd: python3 analysis/cve_surv_analysis.py --eval-path=eval/ --path-prefix=''
      data/cve_survival_input_df
    deps:
    - path: analysis/cve_surv_analysis.py
      md5: 63dd2aba7d6164c0d01605871a371063
      size: 32766
    - path: data/cve_survival_input_df
      md5: 3aae64bcf49544994d0936a549e2517b
      size: 9076263
    params:
      params.yaml:
        eval.bootstrap_samples: 50
        eval.confidence: 0.95
        eval.cve_survival_analysis:
          risk_column_name: embargo_max
        eval.description: no additional filtering
    outs:
    - path: eval/cve_surv_group_metrics.csv
      md5: b6d9c84911f7f45a3d120c295ced2407
      size: 229
    - path: eval/cve_surv_metrics.json
      md5: 35cb3e08d243bf19da7289a6fa238122
      size: 461
    - path: eval/cve_surv_params.yaml
      md5: 389fb2ae1ac2e20fc14155970d10e4f5
      size: 330
    - path: eval/cve_surv_statistics.csv
      md5: dd0d9778a2613b2058a05bc28d3f0c65
      size: 234
    - path: eval/cve_survival_function.png
      md5: 7a9fa45ca5852b69b6ef3ac29cf6488a
      size: 35052
  merge_project_info_into_lifespan_language:
    cmd: python3 projects_stats/merge_with_project_metadata.py data/cve_lifespan_language_and_cve_info_df  data/unique_project_info.parquet
      data/cve_survival_input_most_used_language_df
    deps:
    - path: data/cve_lifespan_language_and_cve_info_df
      md5: 0107e90bacf138012923ee2d3b4be23c
      size: 3738093
    - path: data/unique_project_info.parquet
      md5: ab9166e42442236878e119a9c8fcfe49
      size: 2664107
    - path: projects_stats/merge_with_project_metadata.py
      md5: b9b55377ae37a1e62e19a0ed49ed208b
      size: 5424
    outs:
    - path: data/cve_survival_input_most_used_language_df
      md5: e826986f18042890257ca93e0874534f
      size: 5687409
  merge_project_info_into_lifespan:
    cmd: python3 projects_stats/merge_with_project_metadata.py data/cve_lifespan_and_cve_info_df
      data/unique_project_info.parquet data/cve_survival_input_df
    deps:
    - path: data/cve_lifespan_and_cve_info_df
      md5: 7dfa44782d07e90c4a46bc8cdd64b355
      size: 5860513
    - path: data/unique_project_info.parquet
      md5: ab9166e42442236878e119a9c8fcfe49
      size: 2664107
    - path: projects_stats/merge_with_project_metadata.py
      md5: b9b55377ae37a1e62e19a0ed49ed208b
      size: 5424
    outs:
    - path: data/cve_survival_input_df
      md5: 3aae64bcf49544994d0936a549e2517b
      size: 9076263
  dependency_df:
    cmd: python3 dependency_search/prepare_dep_df.py data/cve_df_filename data/dep_df
    deps:
    - path: data/cve_df_filename
      md5: 2edd1805e47670ff381b6e81f5b03463
      size: 60583695
    - path: dependency_search/prepare_dep_df.py
      md5: e2493ae952ee56ccb4b0e2779ae58cf9
      size: 834
    outs:
    - path: data/dep_df
      md5: 462062bf333cd25670e3e65a9abb980f
      size: 30932536