Consuming paginated API using periodic Celery tasks in Django

				...
import logging
import time

import requests
from celery import shared_task
from django.conf import settings

from core.models import FullCoin

logger = logging.getLogger(__name__)


def build_api_url(page: int) -> str:
    """Build the API URL."""
    market_currency_order = 'markets?vs_currency=usd&order=market_cap_desc&'
    per_page = f'per_page=50&page={page}&sparkline=false'
    return f'{settings.BASE_API_URL}/coins/{market_currency_order}{per_page}'


def fetch_coins_iteratively() -> Generator[dict, None, None]:
    """Fetch coins data from API using generator."""
    page = 1
    while True:
        try:
            url = build_api_url(page)
            response = requests.get(url)
            coin_data = response.json()

            # Check for rate limit response
            if isinstance(coin_data, dict) and coin_data.get('status', {}).get('error_code') == 429:
                logger.warning("Rate limit exceeded. Waiting 60 seconds...")
                time.sleep(60)
                continue

            # Check for empty response (end of pagination)
            if not coin_data:
                break

            yield from coin_data
            logger.info(f"Fetched page {page} with {len(coin_data)} coins")
            page += 1

            time.sleep(1)  # Be nice to the API

        except requests.exceptions.RequestException as e:
            logger.error(f"Request failed on page {page}: {e}")
            raise


@shared_task(
    bind=True,
    autoretry_for=(Exception,),
    retry_backoff=True,
    retry_backoff_max=600,
    max_retries=5,
)
def get_full_coin_data_iteratively_for_page(self) -> None:
    """Get full coin data iteratively for each page."""
    try:
        # Use list comprehension to collect coins in batches
        batch_size = 100
        coins_batch = []

        for coin in fetch_coins_iteratively():
            coins_batch.append(coin)

            if len(coins_batch) >= batch_size:
                logger.info(f"Processing batch of {len(coins_batch)} coins")
                store_data(coins_batch)
                coins_batch = []

        # Process remaining coins
        if coins_batch:
            logger.info(f"Processing final batch of {len(coins_batch)} coins")
            store_data(coins_batch)

    except Exception as e:
        logger.error(f"Failed to process coins: {e}")
        raise self.retry(exc=e)