Create and retrieve data exports

import requests                 # needs to be installed
import json
import time
from zipfile import ZipFile
from io import BytesIO
import pandas as pd             # needs to be installed
​
​
def create_export(token, from_date, to_date, org=None, file_name=None, d=None):
    """Creates an export
​
    :param token: An access token granting the Analytics - Data exports - Edit privilege in the target organization.
    :param from_date: The date to begin the export at (for example, "2019-03-12T14:20:20.266Z").
    :param to_date: The date to end the export at (for example, "2019-04-12T14:20:20.266Z").
    :param org: The organization ID; only required when authenticating with an OAuth2 token.
    :param file_name: The export name; defaults to the generated export ID.
    :param d: A description of the export.
    :return: The generated export ID.
    """
​
    url = "https://platform.cloud.coveo.com/rest/ua/v15/exports"
​
    # define export
    querystring = {
        "from": from_date,
        "to": to_date
    }
​
    if org:
        querystring["org"] = org
    if file_name:
        querystring["filename"] = file_name
    if d:
        querystring["d"] = d
​
    headers = {
        "Authorization": "Bearer %s" % token
    }
​
    response = requests.request("POST", url, headers=headers, params=querystring)
​
    response_json = json.loads(response.text)
    try:
        return response_json["id"]
    except:
        print("\nThere was an issue with your export creation request. Here is the response from the service:")
        print("    %s\n" % json.dumps(response_json))
​
​
def get_export(id, token, org=None):
    """Shows the information of an export
​
    :param id: The export ID.
    :param token: An access token granting the Analytics - Data exports - View privilege in the target organization.
    :param org: The organization ID; only required when authenticating with an OAuth2 token.
    :return: Information about the export in JSON format.
    """
​
    # request to get export link
    url = "https://platform.cloud.coveo.com/rest/ua/v15/exports/%s" % id
​
    querystring = {
        "redirect": "false"
    }
​
    if org:
        querystring["org"] = org
​
    headers = {
        "Authorization": "Bearer %s" % token
    }
​
    response = requests.request("GET", url, headers=headers, params=querystring)
    response_json = json.loads(response.text)
​
    return response_json
​
​
def download_export(link, token, org=None):
    """Downloads an export
​
    :param link: The export link. Contains the id and redirect parameter.
    :param token: An access token granting the Analytics - Data exports - View privilege in the target organization.
    :param org: The organization ID; only required when authenticating with an OAuth2 token.
    """
​
    if org:
        link += "&org=%s" % org
​
    headers = {
        "Authorization": "Bearer %s" % token
    }
​
    # get the export using the link which already contains the appropriate query parameters, and possibly the org
    data = requests.get(link, headers=headers, stream=True)
​
    z = ZipFile(BytesIO(data.content))
​
    z.extractall('')
​
​
def join_tables():
    """Joins the clicks, searches, and custom_events tables in the current repository.
    """
​
    # get click, search, and custom events
    data = [pd.read_csv(csv_file) for csv_file in ["clicks.csv", "searches.csv", "custom_events.csv"]]
​
    # add c_eventType column
    df = [pd.DataFrame(table) for table in data]
    df[0]["c_eventType"] = "Click"
    df[1]["c_eventType"] = "Search"
    df[2]["c_eventType"] = "Custom"
​
    # join DataFrames into one
    dfAllEvents = pd.concat(df, sort=True)
    # sort first by visitId and then by datetime
    dfAllEvents = dfAllEvents.sort_values(by=["visitId", "datetime"], ascending=True)
    # reset the index column
    dfAllEvents = dfAllEvents.reset_index(drop=True)
    # export to csv
    dfAllEvents.to_csv("allEvents.csv")
​
​
if __name__ == "__main__":
​
    token = "**********-****-****-****-***********"
    from_date = "2018-06-01T05:00:00.000Z"
    to_date = "2019-03-01T04:59:59.999Z"
    org = None
​
    export_id = create_export(token, from_date, to_date, org=org)
​
    # initial time to wait in between queries when checking the status of the export
    waitInterval = 4
    timeCounter = waitInterval
    status = "PENDING"
    while status != "AVAILABLE":
        time.sleep(waitInterval)
        print("%s ... %s seconds" % (status, timeCounter))
        # exponential backoff, until 60 seconds
        waitInterval = 2 * waitInterval if waitInterval < 60 else waitInterval
        timeCounter += waitInterval
        export = get_export(export_id, token, org=org)
        try:
            status = export["status"]
        except:
            print("\nThere was an issue with your export get request. Here is the response from the service:")
            print("    %s\n" % json.dumps(export))
            break
​
    try:
        download_export(export["downloadLink"], token, org=org)
        print("Export complete")
    except:
        print("\nThere was an issue when attempting to download your export.\n")
​
​
    join_tables()