enqueue-listings.py

import os, logging.handlers, sys
from syslog_rfc5424_formatter import RFC5424Formatter
from adapters import ResponseEnvelope, Listing, HoursFilter
from api import ListHubClient
from rq import Connection, Queue, Retry
from workers import property_media_worker
from redis import Redis
from notifiers import init_sentry
from dotenv import load_dotenv

load_dotenv()

init_sentry()

syslogger = logging.getLogger("listhub-data-capture")
syslogger.setLevel(logging.INFO)

handler = logging.handlers.SysLogHandler(facility=2, address="/dev/log")
handler.setFormatter(RFC5424Formatter())
syslogger.addHandler(handler)

q = Queue(
    sys.argv[1], connection=Redis(unix_socket_path="/var/run/redis/redis-server.sock")
)


def browse(listings, query_type):
    for l in listings:
        for m in l.media:
            enqueue_media_link(m, l, query_type)

def enqueue_media_link(m, l, query_type):
    listing_key = l.listing_key
    media_key = m["MediaKey"]
    media_url = m["MediaURL"]

    syslogger.info(f"Enqueing {listing_key}-{media_key} with timestamp {l.last_update}")

    q.enqueue(
        property_media_worker,
        listing_key,
        media_key,
        media_url,
        query_type,
        result_ttl=0,
        retry=Retry(max=3, interval=60),
    )


if __name__ == "__main__":
    query_type = sys.argv[1]
    hours_ago = int(sys.argv[2])

    syslogger.info(f"{sys.argv[0]} using {query_type}-{hours_ago} as filter")

    client = ListHubClient.create(query_type)

    next_page_url = None

    counter = 0

    while True:
        counter = counter + 1

        try:
            envelope = ResponseEnvelope(client.get_listings(next_page_url))

            browse(
                HoursFilter(envelope.listings, hours_ago=hours_ago).results, query_type
            )

            next_page_url = envelope.next_page_url

        except KeyError:
            syslogger.error("Key error Encountered - looping")

        if not next_page_url:
            break