-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathimport_data.py
128 lines (117 loc) · 3.5 KB
/
import_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
from logging import exception
from utile.database import (
config_parse,
create_table,
create_cnx,
inject_into_db,
delete_duplicates,
create_tables_vdsa,
inject_into_db_vdsa,
)
import argparse
import os
list_tables = [
"Crops_Area_Production",
"Crop_Wise_Irrigated_Area",
"HYV_Area_Cereal_Crop",
"Land_Use",
"Source_Wise_Irrigated_Area",
"Cropped_And_Irrigated_Area",
"Fodder_And_Irrigated_Area",
"Farm_Havest_Price",
"Fertilizer_Consumption",
"Fertilizer_Prices",
"Annual_Monthly_Actual_Rainfall",
"Population_Census",
"Livestock_Census",
"Agriculture_Implements_Census",
"Market_And_Roads",
"Wage_Rate",
"Operation_Holding",
"Annal_Monthly_Normal_Rainfall",
"Length_Growing_Period",
"Soil_Type",
"Annual_Monthly_Normal_Potential_Evapotranspiration",
"Annual_Moisture_Available",
"Agroecological_Subregion",
]
def main() -> None:
"""
This function's aim is to fill the database with excel file
returns -> None
"""
parser = argparse.ArgumentParser(description="Choose to import database")
parser.add_argument(
"--type",
# "typeImport",
metavar="t",
type=str,
# nargs="+",
help="what kind of imported you want to do: base is for the dataset provided by SCOR, external is VDSA dataset, both is for both",
choices=["base", "external", "all"],
default="base",
)
args = parser.parse_args()
# print(args.accumulate(args.integers))
path = "./data"
path_vdsa = "./dataset vdsa"
# Get the configuration of the data
parser_config = config_parse()
# Connect to database
connexion = create_cnx(parser_config)
cnx = connexion["cnx"]
curs = cnx.cursor()
if args.type in ["base", "all"]:
# Drop former table if necessary
print("base")
try:
curs.execute("DROP TABLE data_SCOR")
print("drop")
except:
pass
# Create table for the data
statement = create_table(path)
curs.execute(statement)
# Inject all the data into db
inject_into_db(path, connexion)
# Following code is really slow
# statement = delete_duplicates(path)
# try:
# curs.execute(statement)
# print("drop duplicates done")
# except Exception as e:
# print("drop duplicates failed")
# print(e)
# Close connexion
if args.type in ["external", "all"]:
print("external")
for table in list_tables:
try:
curs.execute(f"DROP TABLE {table}")
except:
pass
template_folder = os.path.join(path_vdsa, "Andhra Pradesh")
list_files = list(os.listdir(template_folder))
# statements = create_tables_vdsa(path_vdsa)
for i, file in enumerate(list_files):
# print(i)
statement = create_tables_vdsa(file, path_vdsa)
try:
curs.execute(statement)
# if i == 12:
# print(statement)
# print()
except Exception as e:
print(file)
print()
print(statement)
print()
print(e)
# exit(1)
# print(statement)
print("end of creating vdsa tables \n")
inject_into_db_vdsa(path_vdsa, connexion)
curs.close()
cnx.close()
if __name__ == "__main__":
main()