-
Notifications
You must be signed in to change notification settings - Fork 6
CM360 API Example
Paul Kenjora edited this page Jul 18, 2024
·
2 revisions
We had a project where the client needed to build a CM360 data warehouse fast. This example shows how to download various CM360 endpoints from account down to creative. The workflow includes a few tricks for appending and limiting API, and applying filters to reduce the size of the calls.
Each one of these are replicated to a table in BigQuery:
- userProfiles/list
- accounts/list - Note append to add profileId.
- subaccounts/list
- advertiserGroups/list
- advertiserGroups/list
- advertiserLandingPages/list
- campaigns/list
- campaignCreativeAssociations/list - Note limit to shorten call for demo.
- ads/list
- sites/list
- directorySites/list
- placements/list
- placementGroups/list
- placementStrategies/list
- creatives/list
- creativeGroups/list
- sizes/list
- creativeFields/list
- creativeFieldValues/list
- browsers/list
- cities/list
- languages/list
- metros/list
- connectionTypes/list
- contentCategories/list
- countries/list
- regions/list
- postalCodes/list
- videoFormats/list
- platformTypes/list
- orders/list
- mobileCarriers/list
- operatingSystems/list
- operatingSystemVersions/list
- remarketingLists/list
- targetingTemplates/list
- targetableRemarketingLists/list
python3 bqflow/bqflow/run.py workflow.json -s $SERVICE_JSON -p $PROJECT_ID
{
"tasks":[
{ "dataset":{
"description":"Create a dataset for bigquery tables.",
"hour":[4],
"auth":"user",
"dataset":"my_cm360"
}},
{ "google_api": {
"description":"Load user profiles.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/userProfiles/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "userProfiles.list",
"kwargs":{},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_UserProfiles"
}
}
}},
{ "google_api": {
"description":"Load accounts.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/accounts/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":false,
"function": "accounts.get",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT DISTINCT profileId, accountId AS id FROM `CM360_UserProfiles` WHERE (ARRAY_LENGTH({accounts}) = 0 OR accountId IN UNNEST({accounts}))",
"parameters":{
"accounts":["7480"]
}
}
},
"append": [
{ "name": "profileId", "type": "STRING", "mode": "REQUIRED" }
],
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_Accounts"
}
}
}},
{ "google_api": {
"description":"Load subaccounts.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/subaccounts/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "subaccounts.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT DISTINCT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_SubAccounts"
}
}
}},
{ "google_api": {
"description":"Load advertisers.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/advertisers/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "advertisers.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT DISTINCT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_Advertisers"
}
}
}},
{ "google_api": {
"description":"Load advertiser groups.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/advertiserGroups/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "advertiserGroups.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_AdvertiserGroups"
}
}
}},
{ "google_api": {
"description":"Load advertiser landing pages.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/advertiserLandingPages/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "advertiserLandingPages.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_AdvertiserLandingPages"
}
}
}},
{ "google_api": {
"description":"Load campaigns.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/campaigns/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "campaigns.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_Campaigns"
}
}
}},
{ "google_api": {
"description":"Load campaign creative associations.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/campaignCreativeAssociations/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "campaignCreativeAssociations.list",
"limit":500,
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT DISTINCT A.profileId, C.id AS campaignId FROM `CM360_Campaigns` AS C LEFT JOIN `CM360_Accounts` AS A ON C.accountId=A.id"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_CampaignCreativeAssociations"
}
}
}},
{ "google_api": {
"description":"Load ads.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/ads/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "ads.list",
"limit":500,
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_Ads"
}
}
}},
{ "google_api": {
"description":"Load sites.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/sites/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "sites.list",
"limit":500,
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_Sites"
}
}
}},
{ "google_api": {
"description":"Load directory sites.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/directorySites/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "directorySites.list",
"limit":500,
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_DirectorySites"
}
}
}},
{ "google_api": {
"description":"Load placements.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/placements/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "placements.list",
"limit":500,
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_Placements"
}
}
}},
{ "google_api": {
"description":"Load placement groups.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/placementGroups/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "placementGroups.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_PlacementGroups"
}
}
}},
{ "google_api": {
"description":"Load placement strategies.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/placementStrategies/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "placementStrategies.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_placementStrategies"
}
}
}},
{ "google_api": {
"description":"Load creatives.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/creatives",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "creatives.list",
"limit":500,
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_Creatives"
}
}
}},
{ "google_api": {
"description":"Load creative groups.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/creativeGroups/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "creativeGroups.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_CreativeGroups"
}
}
}},
{ "google_api": {
"description":"Load sizes.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/sizes/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "sizes.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_sizes"
}
}
}},
{ "google_api": {
"description":"Load creative fields.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/creativeFields/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "creativeFields.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts`;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_CreativeFields"
}
}
}},
{ "google_api": {
"description":"Load creative field values.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/creativeFieldValues/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "creativeFieldValues.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT A.profileId, C.id AS creativeFieldId FROM `CM360_creativeFields` AS C LEFT JOIN `CM360_Accounts` AS A ON C.accountId=A.id;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_CreativeFieldValues"
}
}
}},
{ "google_api": {
"description":"Load browsers.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/browsers/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "browsers.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts` LIMIT 1;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_Browsers"
}
}
}},
{ "google_api": {
"description":"Load cities.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/cities/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "cities.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts` LIMIT 1;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_Cities"
}
}
}},
{ "google_api": {
"description":"Load languages.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/languages/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "languages.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts` LIMIT 1;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_Languages"
}
}
}},
{ "google_api": {
"description":"Load metros.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/metros/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "metros.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts` LIMIT 1;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_Metros"
}
}
}},
{ "google_api": {
"description":"Load connection types.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/connectionTypes/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "connectionTypes.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts` LIMIT 1;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_ConnectionTypes"
}
}
}},
{ "google_api": {
"description":"Load content categories.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/contentCategories/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "contentCategories.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts` LIMIT 1;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_ContentCategories"
}
}
}},
{ "google_api": {
"description":"Load countries.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/countries/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "countries.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts` LIMIT 1;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_Countries"
}
}
}},
{ "google_api": {
"description":"Load regions.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/regions/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "regions.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts` LIMIT 1;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_Regions"
}
}
}},
{ "google_api": {
"description":"Load postal codes.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/postalCodes/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "postalCodes.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts` LIMIT 1;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_PostalCodes"
}
}
}},
{ "google_api": {
"description":"Load video formats.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/videoFormats/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "videoFormats.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts` LIMIT 1;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_VideoFormats"
}
}
}},
{ "google_api": {
"description":"Load platform types.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/platformTypes/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "platformTypes.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts` LIMIT 1;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_PlatformTypes"
}
}
}},
{ "google_api": {
"description":"Load orders.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/orders/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "orders.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT DISTINCT A.profileId, P.id AS projectId FROM `CM360_Projects` AS P LEFT JOIN `CM360_Accounts` AS A ON P.accountId=A.id"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_Orders"
}
}
}},
{ "google_api": {
"description":"Load mobile carriers.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/mobileCarriers/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "mobileCarriers.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts` LIMIT 1;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_MobileCarriers"
}
}
}},
{ "google_api": {
"description":"Load operating systems.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/operatingSystems/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "operatingSystems.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts` LIMIT 1;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_OperatingSystems"
}
}
}},
{ "google_api": {
"description":"Load operating system versions.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/operatingSystemVersions/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "operatingSystemVersions.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT profileId FROM `CM360_Accounts` LIMIT 1;"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_OperatingSystemVersions"
}
}
}},
{ "google_api": {
"description":"Load remarketing lists.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/remarketingLists/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "remarketingLists.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT id AS advertiserId, id AS accountId FROM `CM360_Accounts` where name='BROKEN API CALL SEE: b/183547271';"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_RemarketingLists"
}
}
}},
{ "google_api": {
"description":"Load targeting templates.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/targetingTemplates/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "targetingTemplates.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT id AS advertiserId, id AS accountId FROM `CM360_Accounts` where name='BROKEN API CALL SEE: b/183547271';"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_TargetingTemplates"
}
}
}},
{ "google_api": {
"description":"Load targetable remarketing lists.",
"documentation":"https://developers.google.com/doubleclick-advertisers/rest/v4/targetableRemarketingLists/list",
"auth":"user",
"api": "dfareporting",
"version": "v4",
"iterate":true,
"function": "targetableRemarketingLists.list",
"kwargs_remote":{
"bigquery":{
"auth": "user",
"dataset":"my_cm360",
"query":"SELECT id AS advertiserId, id AS accountId FROM `CM360_Accounts` where name='BROKEN API CALL SEE: b/183547271';"
}
},
"results": {
"bigquery": {
"auth":"user",
"dataset":"my_cm360",
"table": "CM360_TargetableRemarketingLists"
}
}
}}
]
}