Generate csv from pagination.

[engineer330426]

I was trying to run the code below, but I can't seem to get the csv to populate correctly.

import csv
import glob
from falconpy import Hosts
from api_var import api_key, api_base, api_secrect

#Return all csv files read into the script as dictionaries
csv_name = ['cid_customer.csv', 'another1.csv', 'another2.csv', 'another3.csv', 'another4.csv', 'another5.csv', 'another6.csv']

def device_list(off: int, limit: int, sort: str):
    """Return a list of all devices for the CID, paginating when necessary."""
    result = falcon.query_devices_by_filter_scroll(limit=limit, offset=off, sort=sort)
    new_offset = 0
    total = 0
    returned_device_list = []
    if result["status_code"] == 200:
        new_offset = result["body"]["meta"]["pagination"]["offset"]
        total = result["body"]["meta"]["pagination"]["total"]
        returned_device_list = result["body"]["resources"]

    return new_offset, total, returned_device_list

def device_detail(aids: list):
    """Return the device_id and agent_version for a list of AIDs provided."""
    result = falcon.get_device_details(ids=aids)
    device_details = []
    if result["status_code"] == 200:
        # return the necessary aspects for device lookup.
        for device in result["body"]["resources"]:
            res = {}
            res["hostname"] = device.get("hostname", None)
            res["device_id"] = device.get("device_id", None)
            res["local_ip"] = device.get("local_ip", None)
            res["external_ip"] = device.get("external_ip", None)
            res["cid"] = device.get("cid", None)
            #res["Customer"] = device.get("Customer", None) This is done via lookup creation on backend, needs to be imported as dictionary/List and read through the results and appended.
            res["platform_name"] = device.get("platform_name", None)
            res["chassis_type_desc"] = device.get("chassis_type_desc", None)
            res["connection_ip"] = device.get("connection_ip", None)
            res["mac_address"] = device.get("mac_address", None)
            device_details.append(res)
    return device_details

#FalconPY hosts function
falcon = Hosts(client_id=api_key,
               client_secret=api_secrect,
               base_url=api_base,
               #member_cid=CHILD
               )

OFFSET = None   # First time the token is null
SORT = "hostname.asc" #Defining a sort method
DISPLAYED = 0   # Running count
TOTAL = 1       # Assume there is at least one
LIMIT = 500     # Quick limit to prove pagination
offset_pos = 0  # Start at the beginning

while offset_pos < TOTAL:
    OFFSET, TOTAL, devices = device_list(OFFSET, LIMIT, SORT)
    offset_pos += LIMIT
    details = device_detail(devices)
    for detail in details:
        DISPLAYED += 1
        new_list= [print(f"{DISPLAYED}:hostname : {detail['hostname']}, aid : {detail['device_id']}, cid : {detail['cid']}, platform_name: {detail['platform_name']}, local_ip : {detail['local_ip']}, external_ip : {detail['external_ip']}, chassis_type_desc : {detail['chassis_type_desc']}")]
 
if not DISPLAYED:
    print("No results returned.")

with open('cs_lookup_host_dev', 'w') as f:
    w = csv.writer(f, delimiter=" ")
    w.writerow(["hostname", "aid", "cid", "platform_name", "local_ip", "external_ip", "chassis_type_desc"])
    w.writerows(new_list)

[jshcodes]

Hi @engineer330426 -

I tweaked your code slightly to move new_list over to leveraging the append method, and to be a list of lists.

Let us know if this works for you?

import csv
import glob
from falconpy import Hosts
from api_var import api_key, api_base, api_secrect

#Return all csv files read into the script as dictionaries
csv_name = ['cid_customer.csv', 'another1.csv', 'another2.csv', 'another3.csv', 'another4.csv', 'another5.csv', 'another6.csv']

def device_list(off: int, limit: int, sort: str):
    """Return a list of all devices for the CID, paginating when necessary."""
    result = falcon.query_devices_by_filter_scroll(limit=limit, offset=off, sort=sort)
    new_offset = 0
    total = 0
    returned_device_list = []
    if result["status_code"] == 200:
        new_offset = result["body"]["meta"]["pagination"]["offset"]
        total = result["body"]["meta"]["pagination"]["total"]
        returned_device_list = result["body"]["resources"]

    return new_offset, total, returned_device_list

def device_detail(aids: list):
    """Return the device_id and agent_version for a list of AIDs provided."""
    result = falcon.get_device_details(ids=aids)
    device_details = []
    if result["status_code"] == 200:
        # return the necessary aspects for device lookup.
        for device in result["body"]["resources"]:
            res = {}
            res["hostname"] = device.get("hostname", None)
            res["device_id"] = device.get("device_id", None)
            res["local_ip"] = device.get("local_ip", None)
            res["external_ip"] = device.get("external_ip", None)
            res["cid"] = device.get("cid", None)
            #res["Customer"] = device.get("Customer", None) This is done via lookup creation on backend, needs to be imported as dictionary/List and read through the results and appended.
            res["platform_name"] = device.get("platform_name", None)
            res["chassis_type_desc"] = device.get("chassis_type_desc", None)
            res["connection_ip"] = device.get("connection_ip", None)
            res["mac_address"] = device.get("mac_address", None)
            device_details.append(res)
    return device_details

#FalconPY hosts function
falcon = Hosts(client_id=api_key,
               client_secret=api_secrect,
               base_url=api_base,
               #member_cid=CHILD
               )

OFFSET = None   # First time the token is null
SORT = "hostname.asc" #Defining a sort method
DISPLAYED = 0   # Running count
TOTAL = 1       # Assume there is at least one
LIMIT = 500     # Quick limit to prove pagination
offset_pos = 0  # Start at the beginning

while offset_pos < TOTAL:
    OFFSET, TOTAL, devices = device_list(OFFSET, LIMIT, SORT)
    offset_pos += LIMIT
    details = device_detail(devices)

    new_list = []
    for detail in details:
        DISPLAYED += 1
        new_list.append([detail["hostname"],
                         detail["device_id"],
                         detail["cid"],
                         detail["platform_name"],
                         detail["local_ip"],
                         detail["external_ip"],
                         detail["chassis_type_desc"]
                         ])
 
if not DISPLAYED:
    print("No results returned.")

print(f"Writing {DISPLAYED} results to CSV.")

with open('cs_lookup_host_dev.csv', 'w') as f:
    w = csv.writer(f, delimiter=" ")
    w.writerow(["hostname", "aid", "cid", "platform_name", "local_ip", "external_ip", "chassis_type_desc"])
    w.writerows(new_list)

[engineer330426]

It works to write the actual results to the file, the problem im having now is that it only writes 498 results out of the 32745 to the csv file.

[engineer330426]

I believe its only printing out the end of the list, because of where the output to the csv is, i moved it up to the for loop hopefully it works better there.

[engineer330426]

I got it, i had to read the file in with reader, and change the switch from a write(w) to an append (a)