-
Notifications
You must be signed in to change notification settings - Fork 0
/
default_spec.json
141 lines (131 loc) · 4.24 KB
/
default_spec.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
{
"application": "Web crawler singer.io tap",
"args":
{
"encoding":
{
"type": "string",
"default": "utf-8",
"help": "file encoding"
},
"skip":
{
"type": "integer",
"default": 0,
"help": "records to skip when inferring schema"
},
"selenium_ide_script":
{
"type": "string",
"help": "Selenium IDE exported script for web-driver instructions"
},
"streams":
{
"type": "string",
"default": null,
"help": "comma-separated strings of tap stream IDs"
},
"username":
{
"type": "string",
"default": null,
"help": "username used for authentication if applicable"
},
"password":
{
"type": "string",
"default": null,
"help": "password used for authentication if applicable"
},
"schema_dir":
{
"type": "string",
"default": "./schema",
"help": "Path to the schema directory"
},
"catalog_dir":
{
"type": "string",
"default": "./catalog",
"help": "Path to the catalog directory"
},
"timestamp_key":
{
"type": "string",
"default": null,
"help": "Unix timestamp key(column) name. If this is not null, timestamp_key is ignored"
},
"start_timestamp":
{
"type": "integer",
"default": null,
"help": "Start Unix timestamp"
},
"end_timestamp":
{
"type": "integer",
"default": null,
"help": "End Unix timestamp. When this is set, tap only replicates the record with end_timestamp younger than end_timestamp. Once equal or greater than end_datetime is read, the process exits."
},
"datetime_key":
{
"type": "string",
"default": null,
"help": "Datetime key(column) name. If this is not null, timestamp_key is ignored"
},
"start_datetime":
{
"type": "datetime",
"default": null,
"help": "Start datetime in ISO 8601 format. As a convenience, this will be automatically converted to timestamp if datetime_key is null and timestamp_key is set and start_timestamp is null."
},
"end_datetime":
{
"type": "datetime",
"default": null,
"help": "End datetime in ISO 8601 format. When this is set, tap only replicates the record with datetime younger than end_datetime. Once equal or greater than end_datetime is read, the process exits. As a convenience, this will be automatically converted to timestamp if datetime_key is null and timestamp_key is set and end_timestamp is null."
},
"index_key":
{
"type": "string",
"default": null,
"help": "Index key (column) name"
},
"start_index":
{
"type": "integer",
"default": 0,
"help": "Starting index number"
},
"end_index":
{
"type": "integer",
"default": 0,
"help": "When this is set, tap only replicates the record with index younger than end_index. Once equal or greater than end_index is read, the process exits."
},
"items_per_page":
{
"type": "integer",
"default": 100,
"help": "# of items per page if API supports paging"
},
"max_page":
{
"type": "integer",
"default": null,
"help": "If set, stop polling after max_page"
},
"record_list_level":
{
"type": "string",
"default": null,
"help": "Set this like 'level_1,level_2...' if the target list is at raw_json_response[level_1][level_2]..."
},
"record_level":
{
"type": "string",
"default": null,
"help": "Set this like 'level_a,level_b...' if the target object is at raw_individual_record[level_a][level_b]..."
}
}
}