Main Code.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L_wX4yZMQxa0"
      },
      "source": [
        "# User Settings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EcLOU9WsQw-_"
      },
      "outputs": [],
      "source": [
        "# What folder do you want to save the results in?\n",
        "# Individual CSV's for each coin saved with coin name as filename\n",
        "RESULTS_FOLDER = 'results'\n",
        "COMBINE_ALL = True"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WPCbkMgDQ8vY"
      },
      "source": [
        "# Setup Notebook"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F1eiojR90fXb",
        "outputId": "1ed9e1ca-0437-472c-c0eb-f00063f9a8a3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: fastcore in /usr/local/lib/python3.7/dist-packages (1.3.27)\n",
            "Requirement already satisfied: pip in /usr/local/lib/python3.7/dist-packages (from fastcore) (21.1.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from fastcore) (21.3)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->fastcore) (3.0.6)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (3.8.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (1.7.2)\n",
            "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (0.13.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (21.2.0)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (2.0.8)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (1.2.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (4.0.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (5.2.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (3.10.0.2)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp) (1.2.0)\n",
            "Requirement already satisfied: idna>=2.0 in /usr/local/lib/python3.7/dist-packages (from yarl<2.0,>=1.0->aiohttp) (2.10)\n"
          ]
        }
      ],
      "source": [
        "!pip install fastcore\n",
        "!pip install aiohttp\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "J-fVLY3YVc-e"
      },
      "outputs": [],
      "source": [
        "from fastprogress import progress_bar\n",
        "import requests \n",
        "from bs4 import BeautifulSoup\n",
        "import pandas as pd\n",
        "import re\n",
        "import humanize\n",
        "from IPython.display import clear_output, display\n",
        "from pathlib import Path\n",
        "import numpy as np\n",
        "from fastcore.all import *\n",
        "from time import sleep\n",
        "import random\n",
        "\n",
        "import asyncio\n",
        "import aiohttp\n",
        "\n",
        "# this is to allow it to run in google colab environment\n",
        "import nest_asyncio\n",
        "nest_asyncio.apply()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DXcICVt5RM6B"
      },
      "outputs": [],
      "source": [
        "Path(RESULTS_FOLDER).mkdir(exist_ok=True, parents=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uMnAGdCZPQK1"
      },
      "source": [
        "#Helper Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3VzQeEpqN7ow"
      },
      "outputs": [],
      "source": [
        "def parse_strlist(sl):\n",
        "    clean = re.sub(\"[\\[\\],\\s]\",\"\",sl)\n",
        "    splitted = re.split(\"[\\'\\\"]\",clean)\n",
        "    values_only = [s for s in splitted if s != '']\n",
        "    return values_only\n",
        "\n",
        "def get_bitinfochart_graph_values(url, var_name):\n",
        "  sleep(random.uniform(0, 1.0))\n",
        "  response = requests.get(url)\n",
        "  soup = BeautifulSoup(response.text, 'html.parser')\n",
        "\n",
        "  scripts = soup.find_all('script')\n",
        "  for script in scripts:\n",
        "      if 'd = new Dygraph(document.getElementById(\"container\")' in script.text:\n",
        "          StrList = script.text\n",
        "          StrList = '[[' + StrList.split('[[')[-1]\n",
        "          StrList = StrList.split(']]')[0] +']]'\n",
        "          StrList = StrList.replace(\"new Date(\", '').replace(')','')\n",
        "          dataList = parse_strlist(StrList)\n",
        "\n",
        "  date = []\n",
        "  value = []\n",
        "  for each in dataList:\n",
        "      if (dataList.index(each) % 2) == 0:\n",
        "          date.append(each)\n",
        "      else:\n",
        "          value.append(each)\n",
        "\n",
        "  df = pd.DataFrame(list(zip(date, value)), columns=[\"date\",var_name])\n",
        "  return df\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BVcqBoD0SRSD"
      },
      "outputs": [],
      "source": [
        "# This function takes a list of df's then merge's them on the date field with an outer join\n",
        "\n",
        "def merge_dfs(df_list):\n",
        "  df_merged = None\n",
        "  for i in range(len(df_list)-1):\n",
        "    if i == 0:\n",
        "      df_merged = df_list[i].merge(df_list[i+1], on='date', how='outer')\n",
        "    else:\n",
        "      df_merged = df_merged.merge(df_list[i+1], on='date', how='outer')\n",
        "\n",
        "  return df_merged\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "45zzLEyIPdqx"
      },
      "source": [
        "# Generate List of Pages to Scrape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Vh6yCLGeOBcw"
      },
      "outputs": [],
      "source": [
        "# Manually copy/pasted to use as a starting point\n",
        "\n",
        "chart_dict_list = [{'url': 'https://bitinfocharts.com/comparison/bitcoin-transactions.html', 'name': 'transactions'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/size-btc.html', 'name': 'block_size'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/sentbyaddress-btc.html', 'name': 'sent_addresses'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/bitcoin-difficulty.html', 'name': 'difficulty'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/bitcoin-hashrate.html', 'name': 'hashrate'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/bitcoin-mining_profitability.html', 'name': 'mining_profitability'},\n",
        "                   \n",
        "                    {'url': 'https://bitinfocharts.com/comparison/sentinusd-btc.html', 'name': 'send_usd'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/bitcoin-transactionfees.html', 'name': 'av_transaction_size'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/bitcoin-median_transaction_fee.html', 'name': 'median_transaction_size'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/bitcoin-confirmationtime.html', 'name': 'confirmation_time'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/bitcoin-marketcap.html', 'name': 'market_cap'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/transactionvalue-btc.html', 'name': 'av_transaction_value'},\n",
        "                   \n",
        "                    {'url': 'https://bitinfocharts.com/comparison/mediantransactionvalue-btc.html', 'name': 'median_transaction_value'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/tweets-btc.html', 'name': 'tweets'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/google_trends-btc.html', 'name': 'google_trends'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/activeaddresses-btc.html', 'name': 'active_addresses'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/top100cap-btc.html', 'name': 'top_100_percent'},\n",
        "                    {'url': 'https://bitinfocharts.com/comparison/fee_to_reward-btc.html', 'name': 'fee_reward'},\n",
        "\n",
        "                    ]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2KHKd7INWJj8",
        "outputId": "da1e8215-dbe1-4dce-c0c7-83f1dacc9f08"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'coin': 'btc', 'full_name': 'bitcoin'},\n",
              " {'coin': 'eth', 'full_name': 'ethereum'},\n",
              " {'coin': 'xrp', 'full_name': 'xrp'}]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "# Get list of all available coins\n",
        "\n",
        "url = 'https://bitinfocharts.com'\n",
        "response = requests.get(url)\n",
        "soup = BeautifulSoup(response.text, 'html.parser')\n",
        "\n",
        "coin_dict_list = []\n",
        "\n",
        "\n",
        "for span in soup.find_all('span'):\n",
        "  \n",
        "  if 's_coins' in str(span.get('class')):\n",
        "    name = span.get('title').lower()\n",
        "    coin = span.get('data-coin')\n",
        "    coin_dict_list.append({'full_name': name,\n",
        "                'coin': coin})\n",
        "\n",
        "coin_dict_list[:3]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CWB7Jr2M6mk_",
        "outputId": "8af48403-bd45-4278-e753-c8326464b58e"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'coin': 'btc',\n",
              "  'full_name': 'bitcoin',\n",
              "  'scrape_details': [{'name': 'transactions',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-transactions.html'},\n",
              "   {'name': 'block_size',\n",
              "    'url': 'https://bitinfocharts.com/comparison/size-btc.html'},\n",
              "   {'name': 'sent_addresses',\n",
              "    'url': 'https://bitinfocharts.com/comparison/sentbyaddress-btc.html'},\n",
              "   {'name': 'difficulty',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-difficulty.html'},\n",
              "   {'name': 'hashrate',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-hashrate.html'},\n",
              "   {'name': 'mining_profitability',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-mining_profitability.html'},\n",
              "   {'name': 'send_usd',\n",
              "    'url': 'https://bitinfocharts.com/comparison/sentinusd-btc.html'},\n",
              "   {'name': 'av_transaction_size',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-transactionfees.html'},\n",
              "   {'name': 'median_transaction_size',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-median_transaction_fee.html'},\n",
              "   {'name': 'confirmation_time',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-confirmationtime.html'},\n",
              "   {'name': 'market_cap',\n",
              "    'url': 'https://bitinfocharts.com/comparison/bitcoin-marketcap.html'},\n",
              "   {'name': 'av_transaction_value',\n",
              "    'url': 'https://bitinfocharts.com/comparison/transactionvalue-btc.html'},\n",
              "   {'name': 'median_transaction_value',\n",
              "    'url': 'https://bitinfocharts.com/comparison/mediantransactionvalue-btc.html'},\n",
              "   {'name': 'tweets',\n",
              "    'url': 'https://bitinfocharts.com/comparison/tweets-btc.html'},\n",
              "   {'name': 'google_trends',\n",
              "    'url': 'https://bitinfocharts.com/comparison/google_trends-btc.html'},\n",
              "   {'name': 'active_addresses',\n",
              "    'url': 'https://bitinfocharts.com/comparison/activeaddresses-btc.html'},\n",
              "   {'name': 'top_100_percent',\n",
              "    'url': 'https://bitinfocharts.com/comparison/top100cap-btc.html'},\n",
              "   {'name': 'fee_reward',\n",
              "    'url': 'https://bitinfocharts.com/comparison/fee_to_reward-btc.html'}]},\n",
              " {'coin': 'eth',\n",
              "  'full_name': 'ethereum',\n",
              "  'scrape_details': [{'name': 'transactions',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-transactions.html'},\n",
              "   {'name': 'block_size',\n",
              "    'url': 'https://bitinfocharts.com/comparison/size-eth.html'},\n",
              "   {'name': 'sent_addresses',\n",
              "    'url': 'https://bitinfocharts.com/comparison/sentbyaddress-eth.html'},\n",
              "   {'name': 'difficulty',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-difficulty.html'},\n",
              "   {'name': 'hashrate',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-hashrate.html'},\n",
              "   {'name': 'mining_profitability',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-mining_profitability.html'},\n",
              "   {'name': 'send_usd',\n",
              "    'url': 'https://bitinfocharts.com/comparison/sentinusd-eth.html'},\n",
              "   {'name': 'av_transaction_size',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-transactionfees.html'},\n",
              "   {'name': 'median_transaction_size',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-median_transaction_fee.html'},\n",
              "   {'name': 'confirmation_time',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-confirmationtime.html'},\n",
              "   {'name': 'market_cap',\n",
              "    'url': 'https://bitinfocharts.com/comparison/ethereum-marketcap.html'},\n",
              "   {'name': 'av_transaction_value',\n",
              "    'url': 'https://bitinfocharts.com/comparison/transactionvalue-eth.html'},\n",
              "   {'name': 'median_transaction_value',\n",
              "    'url': 'https://bitinfocharts.com/comparison/mediantransactionvalue-eth.html'},\n",
              "   {'name': 'tweets',\n",
              "    'url': 'https://bitinfocharts.com/comparison/tweets-eth.html'},\n",
              "   {'name': 'google_trends',\n",
              "    'url': 'https://bitinfocharts.com/comparison/google_trends-eth.html'},\n",
              "   {'name': 'active_addresses',\n",
              "    'url': 'https://bitinfocharts.com/comparison/activeaddresses-eth.html'},\n",
              "   {'name': 'top_100_percent',\n",
              "    'url': 'https://bitinfocharts.com/comparison/top100cap-eth.html'},\n",
              "   {'name': 'fee_reward',\n",
              "    'url': 'https://bitinfocharts.com/comparison/fee_to_reward-eth.html'}]}]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "# Combine the list of coins with the list of url's to create a master dictionary with coins and url's to scrape\n",
        "\n",
        "for coin_dict in coin_dict_list:\n",
        "  coin_dict['scrape_details'] = []\n",
        "  for chart_dict in chart_dict_list:\n",
        "    temp_dict = chart_dict.copy()\n",
        "    \n",
        "    url = temp_dict['url']\n",
        "    url = url.replace('bitcoin', coin_dict['full_name'])\n",
        "    url = url.replace('btc', coin_dict['coin'])\n",
        "    url = url.replace(' ', '%20')\n",
        "\n",
        "    temp_dict['url'] = url\n",
        "    coin_dict['scrape_details'].append(temp_dict)\n",
        "\n",
        "coin_dict_list[:2]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i3rmddoTRGBP"
      },
      "source": [
        "# Synchronous Code Run\n",
        "\n",
        "- Not recommended\n",
        "- takes about 20 minutes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "id": "Ef0bby4903VM",
        "outputId": "9194c3a4-372e-47e0-f9bd-4296feab696a"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='1' class='' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      100.00% [1/1 00:36<00:00]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "coin_merged_df_list = []\n",
        "\n",
        "for coin_dict in progress_bar(coin_dict_list[:1]):\n",
        "  print(f\"Processing - {coin_dict['full_name']}\")\n",
        "  coin_df_list = []\n",
        "  for page in progress_bar(coin_dict['scrape_details']):\n",
        "    try:\n",
        "      coin_df_list.append(get_bitinfochart_graph_values(url=page['url'], var_name=page['name']))\n",
        "    except:\n",
        "      empty_df = pd.DataFrame()\n",
        "      empty_df['full_name'] = coin_dict['full_name']\n",
        "      empty_df['coin'] = coin_dict['coin']\n",
        "      coin_df_list.append(pd.DataFrame)\n",
        "      print(f\"Error with {coin_dict['full_name']}\")\n",
        "\n",
        "  coin_df = merge_dfs(coin_df_list)\n",
        "  coin_df['full_name'] = coin_dict['full_name']\n",
        "  coin_df['coin'] = coin_dict['coin']\n",
        "\n",
        "  coin_merged_df_list.append(coin_df)\n",
        "\n",
        "  clear_output()\n",
        "\n",
        "  if not COMBINE_ALL:\n",
        "    file_path = RESULTS_FOLDER + '/' +coin_dict['full_name'] + '.csv'\n",
        "    coin_df.to_csv(file_path)\n",
        "  \n",
        "if COMBINE_ALL:\n",
        "  combined_df = pd.concat(coin_merged_df_list, ignore_index=True, sort=False)\n",
        "  combined_df.to_csv(RESULTS_FOLDER + '/' + 'all_bitcoininfochart_data.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kz9pP3zPEcin",
        "outputId": "f151d79a-0abe-4640-cd5a-1a86034a3f7a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "litecoin    3370\n",
              "xrp         3134\n",
              "ethereum    1978\n",
              "Name: full_name, dtype: int64"
            ]
          },
          "execution_count": 24,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "combined_df['full_name'].value_counts()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CoVeCJkEcsj-"
      },
      "source": [
        "# Async Helpers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vjGg_JxScxiY"
      },
      "outputs": [],
      "source": [
        "async def fetch(session, url, full_name, coin, var_name):\n",
        "\n",
        "  try:\n",
        "    async with session.get(url) as resp:\n",
        "        return await resp.text(), full_name, coin, var_name\n",
        "  except:\n",
        "    return 'failed', full_name, coin, var_name\n",
        "        # Catch HTTP errors/exceptions here\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_wweoRbEdYtz"
      },
      "outputs": [],
      "source": [
        "async def fetch_concurrent(coin_dictionary):\n",
        "  page_results = []\n",
        "  loop = asyncio.get_event_loop()\n",
        "\n",
        "\n",
        "  async with aiohttp.ClientSession() as session:\n",
        "    tasks = []\n",
        "    for coin_dict in coin_dictionary:\n",
        "      for page in coin_dict['scrape_details']:\n",
        "\n",
        "        tasks.append(loop.create_task(fetch(session, \n",
        "                                            page['url'], \n",
        "                                            coin_dict['full_name'], \n",
        "                                            coin_dict['coin'],\n",
        "                                            page['name'])))\n",
        "\n",
        "    for result in asyncio.as_completed(tasks):\n",
        "      text, full_name, coin, var_name = await result\n",
        "      result_dict = {'response_text': text,\n",
        "                     'full_name': full_name,\n",
        "                     'coin': coin,\n",
        "                     'var_name': var_name}\n",
        "      page_results.append(result_dict)\n",
        "            \n",
        "      \n",
        "    return page_results\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "L4d9BQ6FoOr5"
      },
      "outputs": [],
      "source": [
        "def extract_info_from_response(response_dict):\n",
        "\n",
        "  response_text = response_dict['response_text']\n",
        "  var_name = response_dict['var_name']\n",
        "  \n",
        "  soup = BeautifulSoup(response_text, 'html.parser')\n",
        "\n",
        "  scripts = soup.find_all('script')\n",
        "  for script in scripts:\n",
        "      if 'd = new Dygraph(document.getElementById(\"container\")' in script.text:\n",
        "          StrList = script.text\n",
        "          StrList = '[[' + StrList.split('[[')[-1]\n",
        "          StrList = StrList.split(']]')[0] +']]'\n",
        "          StrList = StrList.replace(\"new Date(\", '').replace(')','')\n",
        "          dataList = parse_strlist(StrList)\n",
        "\n",
        "  date = []\n",
        "  value = []\n",
        "  for each in dataList:\n",
        "      if (dataList.index(each) % 2) == 0:\n",
        "          date.append(each)\n",
        "      else:\n",
        "          value.append(each)\n",
        "\n",
        "  df = pd.DataFrame(list(zip(date, value)), columns=[\"date\",var_name])\n",
        "\n",
        "\n",
        "  if df[var_name].str.isnumeric().any():\n",
        "    if df[var_name].str.contains('nullnull').any():\n",
        "      df[var_name] = np.nan\n",
        "\n",
        "    return df\n",
        "  \n",
        "  else:\n",
        "    return pd.DataFrame(columns=['date', var_name])\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZxMocesf2RLv"
      },
      "source": [
        "# Async Code Run\n",
        "\n",
        "- Takes about 13s for all the requests\n",
        "- then takes 6 minutes to convert everything to dataframes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oUU3XSUBjXyQ",
        "outputId": "0b87a6b1-129c-4890-8043-136dc68ca15e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CPU times: user 4.44 s, sys: 614 ms, total: 5.06 s\n",
            "Wall time: 13.4 s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "http_responses = asyncio.run(fetch_concurrent(coin_dict_list))\n",
        "len(http_responses)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "id": "jHeViMYdlh8w",
        "outputId": "e9a95545-c478-4ba4-e1ef-5e6f388f627e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='45' class='' max='45' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      100.00% [45/45 04:00<00:00]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "coin_merged_df_list = []\n",
        "\n",
        "for coin_dict in progress_bar(coin_dict_list):\n",
        "  print(f\"Now Processing - {coin_dict['full_name']}\")\n",
        "  coin_df_list = []\n",
        "  for response in progress_bar(http_responses):\n",
        "    if coin_dict['full_name'] == response['full_name']:\n",
        "      coin_df_list.append(extract_info_from_response(response))\n",
        "  \n",
        "  coin_df = merge_dfs(coin_df_list)\n",
        "  coin_df['full_name'] = coin_dict['full_name']\n",
        "  coin_df['coin'] = coin_dict['coin']\n",
        "  coin_merged_df_list.append(coin_df)\n",
        "\n",
        "  file_path = RESULTS_FOLDER + '/' +coin_dict['full_name'] + '__' + coin_dict['coin'] + '.csv'\n",
        "  coin_df.to_csv(file_path)\n",
        "\n",
        "  clear_output()\n",
        "\n",
        "      \n",
        "combined_df = pd.concat(coin_merged_df_list, ignore_index=True, sort=False)\n",
        "\n",
        "combined_df.to_csv('all_bitcoininfochart_data.csv')\n",
        "combined_df.to_pickle('all_bitcoininfochart_data.pkl')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hlnEiNdiuiRd",
        "outputId": "cc0752b2-8918-4057-9e08-92b5fab3d673"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Creating Zip File...\n",
            "Complete!\n"
          ]
        }
      ],
      "source": [
        "print('Creating Zip File...')\n",
        "!zip -r -q results.zip {RESULTS_FOLDER}\n",
        "print('Complete!')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1UHTkjySCYYF"
      },
      "outputs": [],
      "source": [
        "df = coin_merged_df_list[52]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8x8ihTAxGKa7"
      },
      "outputs": [],
      "source": [
        "df_columns = df.columns.to_list()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 669
        },
        "id": "867xWCtiGLUs",
        "outputId": "c4c6e22b-48e3-460a-ddb0-b9dafb297eea"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>full_name</th>\n",
              "      <th>coin</th>\n",
              "      <th>values</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>bitcoin</td>\n",
              "      <td>btc</td>\n",
              "      <td>36708</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>litecoin</td>\n",
              "      <td>ltc</td>\n",
              "      <td>30004</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>dogecoin</td>\n",
              "      <td>doge</td>\n",
              "      <td>22697</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>dash</td>\n",
              "      <td>dash</td>\n",
              "      <td>22538</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>vertcoin</td>\n",
              "      <td>vtc</td>\n",
              "      <td>20239</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>reddcoin</td>\n",
              "      <td>rdd</td>\n",
              "      <td>19494</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>blackcoin</td>\n",
              "      <td>blk</td>\n",
              "      <td>19319</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>feathercoin</td>\n",
              "      <td>ftc</td>\n",
              "      <td>18740</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ethereum</td>\n",
              "      <td>eth</td>\n",
              "      <td>15315</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>ethereum classic</td>\n",
              "      <td>etc</td>\n",
              "      <td>14422</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>monero</td>\n",
              "      <td>xmr</td>\n",
              "      <td>13469</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>zcash</td>\n",
              "      <td>zec</td>\n",
              "      <td>11990</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>bitcoin gold</td>\n",
              "      <td>btg</td>\n",
              "      <td>11227</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>bitcoin cash</td>\n",
              "      <td>bch</td>\n",
              "      <td>11107</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>xrp</td>\n",
              "      <td>xrp</td>\n",
              "      <td>6646</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>bitcoin sv</td>\n",
              "      <td>bsv</td>\n",
              "      <td>6406</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>potcoin</td>\n",
              "      <td>potcoin</td>\n",
              "      <td>3979</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>syscoin</td>\n",
              "      <td>syscoin</td>\n",
              "      <td>3864</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>digibyte</td>\n",
              "      <td>digibyte</td>\n",
              "      <td>2610</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>vericoin</td>\n",
              "      <td>vericoin</td>\n",
              "      <td>2508</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           full_name      coin  values\n",
              "0            bitcoin       btc   36708\n",
              "3           litecoin       ltc   30004\n",
              "10          dogecoin      doge   22697\n",
              "7               dash      dash   22538\n",
              "14          vertcoin       vtc   20239\n",
              "12          reddcoin       rdd   19494\n",
              "16         blackcoin       blk   19319\n",
              "17       feathercoin       ftc   18740\n",
              "1           ethereum       eth   15315\n",
              "9   ethereum classic       etc   14422\n",
              "6             monero       xmr   13469\n",
              "8              zcash       zec   11990\n",
              "11      bitcoin gold       btg   11227\n",
              "4       bitcoin cash       bch   11107\n",
              "2                xrp       xrp    6646\n",
              "5         bitcoin sv       bsv    6406\n",
              "41           potcoin   potcoin    3979\n",
              "23           syscoin   syscoin    3864\n",
              "18          digibyte  digibyte    2610\n",
              "39          vericoin  vericoin    2508"
            ]
          },
          "metadata": {
            "tags": []
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "results_summary_dict_list = []\n",
        "\n",
        "for df in coin_merged_df_list:\n",
        "  full_name = df.loc[0,'full_name']\n",
        "  coin = df.loc[0,'coin']\n",
        "\n",
        "  total_count = 0\n",
        "  df_columns = df.columns.to_list()\n",
        "\n",
        "  for column in df_columns:\n",
        "    if column != 'date':\n",
        "      try:\n",
        "        col_count = df[column].str.isnumeric().sum()\n",
        "        total_count = col_count + total_count\n",
        "      except:\n",
        "        pass\n",
        "  results = {'full_name': full_name,\n",
        "             'coin': coin,\n",
        "             'values': total_count}\n",
        "  results_summary_dict_list.append(results)\n",
        "\n",
        "df_summary = pd.DataFrame(results_summary_dict_list)\n",
        "display(df_summary.sort_values('values', ascending=False).head(20))\n",
        "df_summary.to_csv('summary.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JqEERHgdHaeY"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "WPCbkMgDQ8vY",
        "uMnAGdCZPQK1",
        "45zzLEyIPdqx",
        "i3rmddoTRGBP"
      ],
      "name": "AlgoTrading - BitInfoChart Scraper",
      "toc_visible": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}