From e73b3ec8f82bdf3a3de7a38b57d53c5ff3ab2ce8 Mon Sep 17 00:00:00 2001 From: "Ryan J. Dillon" Date: Tue, 25 Feb 2020 14:47:35 +0100 Subject: [PATCH] Add more paths and prefixes (#948) * Add paths and prefixes to resolve on datalake * Reorder asset paths with ascending PlantIDs --- .../dataset/data_provider/ncs_reader.py | 37 ++++++++++++++++--- gordo/machine/dataset/sensor_tag.py | 31 +++++++++------- 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/gordo/machine/dataset/data_provider/ncs_reader.py b/gordo/machine/dataset/data_provider/ncs_reader.py index 8b073dbde..75f218a6c 100644 --- a/gordo/machine/dataset/data_provider/ncs_reader.py +++ b/gordo/machine/dataset/data_provider/ncs_reader.py @@ -18,25 +18,50 @@ class NcsReader(GordoBaseDataProvider): ASSET_TO_PATH = { - "1101-sfb": "/raw/corporate/IMS Statfjord/sensordata/1101-SFB", + # Paths on the datalake with problematic tag naming schemes (e.g. misplaced, + # varying delimiters, non-unique, etc.) are commented with their assioted tag prefixes + "1100-sfa": "/raw/corporate/IMS Statfjord/sensordata/1100-SFA", # None + "1101-sfb": "/raw/corporate/IMS Statfjord/sensordata/1101-SFB", # None + "1102-sfc": "/raw/corporate/IMS Statfjord/sensordata/1102-SFC", # None "1110-gfa": "/raw/corporate/Aspen MS - IP21 Gullfaks A/sensordata/1110-GFA", "1111-gfb": "/raw/corporate/Aspen MS - IP21 Gullfaks B/sensordata/1111-GFB", "1112-gfc": "/raw/corporate/Aspen MS - IP21 Gullfaks C/sensordata/1112-GFC", + "1120-vfr": "/raw/corporate/Aspen MS - IP21 Veslefrikk/sensordata/1120-VFR", "1125-kvb": "/raw/corporate/PI System Operation Norway/sensordata/1125-KVB", "1130-troa": "/raw/corporate/Aspen MS - IP21 Troll A/sensordata/1130-TROA", "1138-val": "/raw/corporate/PI System Operation Norway/sensordata/1138-VAL", - "1163-gdr": "/raw/corporate/PI System Manager Sleipner/sensordata/1163-GDR", - "1175-kri": "/raw/corporate/PI System Operation North/sensordata/1175-KRI", + "1140-sla": "/raw/corporate/PI System Manager Sleipner/sensordata/1140-SLA", # None + "1141-slt": "/raw/corporate/PI System Manager Sleipner/sensordata/1141-SLT", # None + "1142-slb": "/raw/corporate/PI System Manager Sleipner/sensordata/1142-SLB", # None + "1163-gdr": "/raw/corporate/PI System Manager Sleipner/sensordata/1163-GDR", # None + "1170-hd": "/raw/corporate/PI System Operation North/sensordata/1170-HD", + "1175-kri": "/raw/corporate/PI System Operation North/sensordata/1175-KRI", # kri. + "1175-kris": "/raw/corporate/PI System Operation North/sensordata/1175-KRIS", # kri. "1180-nor": "/raw/corporate/PI System Operation North/sensordata/1180-NOR", "1190-asga": "/raw/corporate/PI System Operation North/sensordata/1190-ASGA", - "1191-asgb": "/raw/corporate/PI System Operation North/sensordata/1191-ASGB", - "1218-gkr": "/raw/corporate/PI System Manager Sleipner/sensordata/1218-GKR", + "1191-asgb": "/raw/corporate/PI System Operation North/sensordata/1191-ASGB", # asga. asgb- + "1192-asgs": "/raw/corporate/PI System Operation North/sensordata/1192-ASGS", # asgb. + "1218-gkr": "/raw/corporate/PI System Manager Sleipner/sensordata/1218-GKR", # None + 1218. + "1219-aha": "/raw/corporate/PI System Operation Mam/sensordata/1219-AHA", + "1220-sna": "/raw/corporate/IMS Snorre A/sensordata/1220-SNA", # None + "1221-snb": "/raw/corporate/IMS Snorre B/sensordata/1221-SNB", # None "1230-vis": "/raw/corporate/Aspen MS - IP21 Visund/sensordata/1230-VIS", - "1295-pera": "/raw/corporate/Aspen MS - IP21 Peregrino/sensordata/1295-PERA", + "1294-pera": "/raw/corporate/Aspen MS - IP21 Peregrino/sensordata/1294-PERA", # per. + "1295-pera": "/raw/corporate/Aspen MS - IP21 Peregrino/sensordata/1295-PERA", # per. + "1298-perb": "/raw/corporate/Aspen MS - IP21 Peregrino/sensordata/1298-PERB", # per. + "1299-per": "/raw/corporate/Aspen MS - IP21 Peregrino/sensordata/1299-PER", # per. + "1340-met": "/raw/corporate/PI System Operation Norway/sensordata/1340-MET", + "1380-sno": "/raw/corporate/Aspen MS - IP21 Hammerfest/sensordata/1380-SNO", # 25haxxx_ "1755-gra": "/raw/corporate/Aspen MS - IP21 Grane/sensordata/1755-GRA", "1760-hea": "/raw/corporate/PI System Operation Norway/sensordata/1760-HEA", + "1765-osc": "/raw/corporate/Aspen MS - IP21 Oseberg C/sensordata/1765-OSC", + "1766-oss": "/raw/corporate/Aspen MS - IP21 Oseberg South/sensordata/1766-OSS", + "1767-ose": "/raw/corporate/Aspen MS - IP21 Oseberg East/sensordata/1767-OSE", + "1772-osa": "/raw/corporate/Aspen MS - IP21 Oseberg Field Center/sensordata/1772-OSA", # osf. + "1774-osd": "/raw/corporate/Aspen MS - IP21 Oseberg Field Center/sensordata/1774-OSD", # osf. "1775-trob": "/raw/corporate/Aspen MS - IP21 Troll B/sensordata/1775-TROB", "1776-troc": "/raw/corporate/Aspen MS - IP21 Troll C/sensordata/1776-TROC", + "1886-mara": "/raw/corporate/PI System Operation Mam/sensordata/1886-MARA", "1900-jsv": "/raw/corporate/PI System Operation Johan Sverdrup/sensordata/1900-JSV", "1901-jsv": "/raw/corporate/PI System Operation Johan Sverdrup/sensordata/1901-JSV", "1902-jsv": "/raw/corporate/PI System Operation Johan Sverdrup/sensordata/1902-JSV", diff --git a/gordo/machine/dataset/sensor_tag.py b/gordo/machine/dataset/sensor_tag.py index e6db9dc7f..9e29af0eb 100644 --- a/gordo/machine/dataset/sensor_tag.py +++ b/gordo/machine/dataset/sensor_tag.py @@ -11,32 +11,35 @@ TagPatternToAsset = namedtuple("TagToAsset", ["tag_regexp", "asset_name"]) TAG_TO_ASSET = [ - TagPatternToAsset(re.compile(r"^asgb.", re.IGNORECASE), "1191-asgb"), - TagPatternToAsset(re.compile(r"^gra.", re.IGNORECASE), "1755-gra"), - TagPatternToAsset(re.compile(r"^1125.", re.IGNORECASE), "1125-kvb"), - TagPatternToAsset(re.compile(r"^trb.", re.IGNORECASE), "1775-trob"), - TagPatternToAsset(re.compile(r"^trc.", re.IGNORECASE), "1776-troc"), - TagPatternToAsset(re.compile(r"^tra.", re.IGNORECASE), "1130-troa"), - TagPatternToAsset(re.compile(r"^1218.", re.IGNORECASE), "1218-gkr"), - TagPatternToAsset(re.compile(r"^per.", re.IGNORECASE), "1295-pera"), + TagPatternToAsset(re.compile(r"^ninenine.+::.+", re.IGNORECASE), "ninenine"), + TagPatternToAsset(re.compile(r"^uon_ef.+::.+", re.IGNORECASE), "uon_ef"), TagPatternToAsset(re.compile(r"^gfa.", re.IGNORECASE), "1110-gfa"), TagPatternToAsset(re.compile(r"^gfb.", re.IGNORECASE), "1111-gfb"), TagPatternToAsset(re.compile(r"^gfc.", re.IGNORECASE), "1112-gfc"), + TagPatternToAsset(re.compile(r"^1125.", re.IGNORECASE), "1125-kvb"), + TagPatternToAsset(re.compile(r"^tra.", re.IGNORECASE), "1130-troa"), + TagPatternToAsset(re.compile(r"^asgb.", re.IGNORECASE), "1191-asgb"), TagPatternToAsset(re.compile(r"^kri.", re.IGNORECASE), "1175-kri"), - TagPatternToAsset(re.compile(r"^hea.", re.IGNORECASE), "1760-hea"), - TagPatternToAsset(re.compile(r"^ninenine.+::.+", re.IGNORECASE), "ninenine"), - TagPatternToAsset(re.compile(r"^uon_ef.+::.+", re.IGNORECASE), "uon_ef"), TagPatternToAsset(re.compile(r"^1138.", re.IGNORECASE), "1138-val"), + TagPatternToAsset(re.compile(r"^hd.", re.IGNORECASE), "1170-hd"), TagPatternToAsset(re.compile(r"^nor.", re.IGNORECASE), "1180-nor"), TagPatternToAsset(re.compile(r"^asga.", re.IGNORECASE), "1190-asga"), + TagPatternToAsset(re.compile(r"^1218.", re.IGNORECASE), "1218-gkr"), + TagPatternToAsset(re.compile(r"^1219.", re.IGNORECASE), "1219-aha"), + TagPatternToAsset(re.compile(r"^vis.", re.IGNORECASE), "1230-vis"), + TagPatternToAsset(re.compile(r"^per.", re.IGNORECASE), "1295-pera"), + TagPatternToAsset(re.compile(r"^gra.", re.IGNORECASE), "1755-gra"), + TagPatternToAsset(re.compile(r"^hea.", re.IGNORECASE), "1760-hea"), + TagPatternToAsset(re.compile(r"^osc.", re.IGNORECASE), "1765-OSC"), + TagPatternToAsset(re.compile(r"^oss.", re.IGNORECASE), "1766-OSS"), + TagPatternToAsset(re.compile(r"^ose.", re.IGNORECASE), "1767-OSE"), + TagPatternToAsset(re.compile(r"^trb.", re.IGNORECASE), "1775-trob"), + TagPatternToAsset(re.compile(r"^trc.", re.IGNORECASE), "1776-troc"), TagPatternToAsset(re.compile(r"^1900.", re.IGNORECASE), "1900-jsv"), TagPatternToAsset(re.compile(r"^1901.", re.IGNORECASE), "1901-jsv"), TagPatternToAsset(re.compile(r"^1902.", re.IGNORECASE), "1902-jsv"), TagPatternToAsset(re.compile(r"^1903.", re.IGNORECASE), "1903-jsv"), TagPatternToAsset(re.compile(r"^1904.", re.IGNORECASE), "1904-jsv"), - TagPatternToAsset(re.compile(r"^vis.", re.IGNORECASE), "1230-vis"), - TagPatternToAsset(re.compile(r"^hd.", re.IGNORECASE), "1170-hd"), - TagPatternToAsset(re.compile(r"^1219.", re.IGNORECASE), "1219-aah"), ]