Breadcrumb_Visualization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Breadcrumb_Visualization.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/snupet/Twittorials/blob/master/Breadcrumb_Visualization.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n4Xzknn6SbfV",
        "colab_type": "text"
      },
      "source": [
        "[1] Installs and imports all necessary libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kSgD7YnHwDcH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "!pip install --upgrade -q requests-html extruct bs4 lxml pprint plotly==4.9.0 "
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AFFDJtESvuOb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pprint\n",
        "import requests\n",
        "from extruct.jsonld import JsonLdExtractor\n",
        "from requests_html import HTMLSession\n",
        "from bs4 import BeautifulSoup\n",
        "import lxml\n",
        "import plotly.express as px\n",
        "import pandas as pd"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "phwSNkD0ShEa",
        "colab_type": "text"
      },
      "source": [
        "[2] Initializes variables"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kycI9b_Q3u53",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Holds all URLs with Breadcrumb Lists attributes\n",
        "allURLs = list()\n",
        "\n",
        "pp = pprint.PrettyPrinter(indent=1)\n",
        "\n",
        "user_input = '' #@param {type:\"string\"}"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fVSNe0WgSkkW",
        "colab_type": "text"
      },
      "source": [
        "[3] Function to find breadcrumb trail and add to dictionary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Di7aSp8C3mfO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Finds all URLs with Breadcrumb Lists, counts how many are found, and adds the URL to a list\n",
        "from collections import defaultdict\n",
        "\n",
        "def findType(data):\n",
        "  \n",
        "  for item in data:\n",
        "    for k, v in item.items():\n",
        "      if k == '@type' and v == 'BreadcrumbList':\n",
        "\n",
        "        itemList = item['itemListElement']\n",
        "\n",
        "        breadcrumbList = defaultdict(set)\n",
        "\n",
        "        for breadcrumb in itemList:\n",
        "\n",
        "          name = breadcrumb[\"item\"][\"name\"]\n",
        "          position = breadcrumb[\"position\"]\n",
        "\n",
        "          breadcrumbList[f\"Position{position}\"].add(name)\n",
        "\n",
        "        allURLs.append(breadcrumbList)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vdOxnS3iSpmK",
        "colab_type": "text"
      },
      "source": [
        "[4] Iterates through sitemap and finds URLs, then gathers the HTML from the URL, and puts the data into the above function."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MgrFw_Az21Ef",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "session = HTMLSession()\n",
        "jslde = JsonLdExtractor()\n",
        "\n",
        "r = session.get(user_input)\n",
        "soup = BeautifulSoup(r.text, 'html.parser')\n",
        "URLs = soup.find_all('loc')\n",
        "print(len(URLs))\n",
        "\n",
        "for url in URLs:\n",
        "  url = url.get_text()\n",
        "  try:\n",
        "    html = session.get(url).text\n",
        "    data = jslde.extract(html)\n",
        "    findType(data)\n",
        "\n",
        "  except lxml.etree.ParserError:\n",
        "    print(\"No HTML found for the page\")\n",
        "  except requests.exceptions.TooManyRedirects:\n",
        "    print(\"Exceeded 30 redirects\")\n",
        "  print(f\"Current URL: {url}\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YsCSy2CG_zf0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "allURLs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4TSlNnFGS1YY",
        "colab_type": "text"
      },
      "source": [
        "[5] Initializes dataframe and builds treemap from dataframe."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sRuf2iAC_zpk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.DataFrame(allURLs)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sdWwVetuQt3Q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CFxnOMTEJhZq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.info()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8YQfhrssQwJN",
        "colab_type": "text"
      },
      "source": [
        "If the dataframe above has more than 4 columns, adjust the code below to include more columns. Simply copy the code, change the X in df2[PositionX] = df2.PositionX.astype(str) to the additional column(s). Another line of code must be added per additional column."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TEAlcVmsKETF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from copy import deepcopy\n",
        "df2 = deepcopy(df)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_A8TevR9J8uV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df2[\"Position1\"] = df2.Position1.astype(str)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QCaxdOI3KZuF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df2[\"Position2\"] = df2.Position2.astype(str)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iIxx5_yBKe8-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df2[\"Position3\"] = df2.Position3.astype(str)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kgJYO3j9Khu5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df2[\"Position4\"] = df2.Position4.astype(str)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "swjfgWsoRCa_",
        "colab_type": "text"
      },
      "source": [
        "Adjust the path=[] to include any additional positions if needed."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8j7G0I5sHKDc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fig = px.treemap(df2, path=['Position1', 'Position2', 'Position3'],\n",
        "                  color_continuous_scale='RdBu')\n",
        "fig.show()"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}