diff --git a/cc-huggingface/README.md b/cc-huggingface/README.md new file mode 100644 index 0000000..1fd1a2d --- /dev/null +++ b/cc-huggingface/README.md @@ -0,0 +1,17 @@ +# Using Common Crawl via Hugging Face + +This folder contains examples on how to use Common Crawl data via the Hugging Face ecosystem (buckets, jobs, ...). + +Currently, the hosting of Common Crawl data on Hugging Face is experimental, i.e., the data is limited to small number of crawls. The primary data distribution channel is still AWS S3. + +Notebooks: + +- cc-index-hf.ipynb +- s3-hf.ipynb +- warcio-hf.ipynb +- cdxt-hf.ipynb (TODO) + +See also: + +- https://huggingface.co/docs/hub/storage-buckets +- https://huggingface.co/docs/hub/storage-buckets-s3 diff --git a/cc-huggingface/cc-index-hf.ipynb b/cc-huggingface/cc-index-hf.ipynb new file mode 100644 index 0000000..7134493 --- /dev/null +++ b/cc-huggingface/cc-index-hf.ipynb @@ -0,0 +1,434 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cd98c682", + "metadata": {}, + "source": [ + "# Common Crawl's URL Index via Hugging Face\n", + "\n", + "[Common Crawl's URL Index](https://commoncrawl.org/url-index) (previously known as Columnar Index) is one of the indexes available for querying the Common Crawl corpus. As the name suggests, it is an index to the WARC files and URLs in the Common Crawl corpus in a columnar format (Apache Parquet). This format of the index is suited to efficient analytical and/or bulk queries of the data, saving time and computing resources.\n", + "\n", + "The index files are also available on the Common Crawl HF bucket: https://huggingface.co/buckets/commoncrawl/commoncrawl\n", + "\n", + "The [duckdb](https://duckdb.org/) can be used to query the index without downloading all the data.\n", + "\n", + "## Setup\n", + "\n", + "Follow the [official instructions](https://duckdb.org/install/) to install the duckdb CLI:\n", + "\n", + "```bash\n", + "curl https://install.duckdb.org | sh\n", + "```\n", + "\n", + "## Schema\n", + "\n", + "Let's first look at the schema of the index:\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "303e1e05", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b]11;?\u0007\u001b[90m┌─────────────────────────────────────────────────────┐\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00m\u001b[1mDescribe\u001b[00m\u001b[90m │\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ │\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_surtkey\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_host_name\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_host_tld\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_host_2nd_last_part\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_host_3rd_last_part\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_host_4th_last_part\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_host_5th_last_part\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_host_registry_suffix\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_host_registered_domain\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_host_private_suffix\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_host_private_domain\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_host_name_reversed\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_protocol\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_port\u001b[90m \u001b[00m\u001b[90minteger\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_path\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00murl_query\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00mfetch_time\u001b[90m \u001b[00m\u001b[90mtimestamp with time zone\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00mfetch_status\u001b[90m \u001b[00m\u001b[90msmallint\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00mfetch_redirect\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00mcontent_digest\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00mcontent_mime_type\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00mcontent_mime_detected\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00mcontent_charset\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00mcontent_languages\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00mcontent_truncated\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00mwarc_filename\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00mwarc_record_offset\u001b[90m \u001b[00m\u001b[90minteger\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00mwarc_record_length\u001b[90m \u001b[00m\u001b[90minteger\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00mwarc_segment\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00mcrawl\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m│ \u001b[00msubset\u001b[90m \u001b[00m\u001b[90mvarchar\u001b[00m\u001b[90m \u001b[00m\u001b[90m│\u001b[00m\u001b[90m\n", + "\u001b[00m\u001b[90m└─────────────────────────────────────────────────────┘\u001b[00m\u001b[90m\n", + "\u001b[00m" + ] + } + ], + "source": [ + "# via CLI\n", + "!duckdb -c \"DESCRIBE FROM read_parquet('https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-17/subset=warc/part-00000-ee7620e8-27a7-4c74-846b-a92b039dae92.c000.gz.parquet')\"" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "62859c62", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "┌────────────────────────────┬─────────────┬─────────┬─────────┬─────────┬─────────┐\n", + "│ column_name │ column_type │ null │ key │ default │ extra │\n", + "│ varchar │ varchar │ varchar │ varchar │ varchar │ varchar │\n", + "├────────────────────────────┼─────────────┼─────────┼─────────┼─────────┼─────────┤\n", + "│ url_surtkey │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ url │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ url_host_name │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ url_host_tld │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ url_host_2nd_last_part │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ url_host_3rd_last_part │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ url_host_4th_last_part │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ url_host_5th_last_part │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ url_host_registry_suffix │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ url_host_registered_domain │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ · │ · │ · │ · │ · │ · │\n", + "│ · │ · │ · │ · │ · │ · │\n", + "│ · │ · │ · │ · │ · │ · │\n", + "│ content_mime_detected │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ content_charset │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ content_languages │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ content_truncated │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ warc_filename │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ warc_record_offset │ INTEGER │ YES │ NULL │ NULL │ NULL │\n", + "│ warc_record_length │ INTEGER │ YES │ NULL │ NULL │ NULL │\n", + "│ warc_segment │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ crawl │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "│ subset │ VARCHAR │ YES │ NULL │ NULL │ NULL │\n", + "└────────────────────────────┴─────────────┴─────────┴─────────┴─────────┴─────────┘\n", + " 32 rows (20 shown) 6 columns\n", + "\n" + ] + } + ], + "source": [ + "# via Python client\n", + "import duckdb\n", + "\n", + "index_file = \"https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-17/subset=warc/part-00000-ee7620e8-27a7-4c74-846b-a92b039dae92.c000.gz.parquet\"\n", + "\n", + "duckdb.sql('INSTALL httpfs; LOAD httpfs;')\n", + "duckdb.sql(\n", + " f\"DESCRIBE FROM read_parquet('{index_file}')\"\n", + ").show()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "bc671f82", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/moc-malteos/repos/commoncrawl/cc-notebooks/cc-huggingface/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 9 index files on HF bucket\n" + ] + } + ], + "source": [ + "# The index uses hive-partitioning for `crawl` and `subset` that are automatically constructed if you would use the S3 index.\n", + "# However, we are using HF via HTTP so we need to manually construct the partitions using glob from the HF API.\n", + "\n", + "from huggingface_hub import hffs\n", + "\n", + "# NOTE: We are taking on a sample of all files (part-0000-*)\n", + "index_files = list(hffs.glob(\"buckets/commoncrawl/commoncrawl/cc-index/table/cc-main/warc/crawl=*/subset=*/part-00000-*.parquet\"))\n", + "\n", + "print(f\"Found {len(index_files):,} index files on HF bucket\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "56059cc0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-17/subset=crawldiagnostics/part-00000-ee7620e8-27a7-4c74-846b-a92b039dae92.c000.gz.parquet',\n", + " 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-17/subset=robotstxt/part-00000-ee7620e8-27a7-4c74-846b-a92b039dae92.c000.gz.parquet',\n", + " 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-17/subset=warc/part-00000-ee7620e8-27a7-4c74-846b-a92b039dae92.c000.gz.parquet',\n", + " 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-21/subset=crawldiagnostics/part-00000-0ddabc35-a36d-4326-aa24-6ee6775acdc0.c000.gz.parquet',\n", + " 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-21/subset=robotstxt/part-00000-0ddabc35-a36d-4326-aa24-6ee6775acdc0.c000.gz.parquet',\n", + " 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-21/subset=warc/part-00000-0ddabc35-a36d-4326-aa24-6ee6775acdc0.c000.gz.parquet',\n", + " 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-25/subset=crawldiagnostics/part-00000-b13edba3-e431-43c6-8915-a9f1c955272b.c000.gz.parquet',\n", + " 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-25/subset=robotstxt/part-00000-b13edba3-e431-43c6-8915-a9f1c955272b.c000.gz.parquet',\n", + " 'https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-25/subset=warc/part-00000-b13edba3-e431-43c6-8915-a9f1c955272b.c000.gz.parquet']" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Rewrite bucket paths to HTTP URLs\n", + "http_index_files = [f.replace(\"buckets/commoncrawl/commoncrawl/\", \"https://huggingface.co/buckets/commoncrawl/commoncrawl/resolve/\") for f in index_files]\n", + "\n", + "http_index_files" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "50410d6c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "┌─────────────────┬──────────────────┬──────────────┐\n", + "│ crawl │ subset │ count_star() │\n", + "│ varchar │ varchar │ int64 │\n", + "├─────────────────┼──────────────────┼──────────────┤\n", + "│ CC-MAIN-2026-25 │ warc │ 6890137 │\n", + "│ CC-MAIN-2026-25 │ robotstxt │ 145762 │\n", + "│ CC-MAIN-2026-21 │ robotstxt │ 158868 │\n", + "│ CC-MAIN-2026-21 │ warc │ 7081731 │\n", + "│ CC-MAIN-2026-25 │ crawldiagnostics │ 1491237 │\n", + "│ CC-MAIN-2026-21 │ crawldiagnostics │ 1554206 │\n", + "│ CC-MAIN-2026-17 │ robotstxt │ 266481 │\n", + "│ CC-MAIN-2026-17 │ warc │ 7165967 │\n", + "│ CC-MAIN-2026-17 │ crawldiagnostics │ 1534027 │\n", + "└─────────────────┴──────────────────┴──────────────┘\n", + "\n" + ] + } + ], + "source": [ + "# Let's see how large our sample is\n", + "duckdb.sql(\n", + " f\"SELECT crawl, subset, COUNT(*) FROM read_parquet({http_index_files!r}, hive_partitioning = true) GROUP BY crawl, subset\"\n", + ").show()" + ] + }, + { + "cell_type": "markdown", + "id": "5da96012", + "metadata": {}, + "source": [ + "## Examples\n", + "\n", + "### Query 1: Counting Pages Per Domain\n", + "\n", + "Time to run the first query! This query counts the number of pages per domain within a single top-level domain, `.ru` (Russia):\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "7df2066c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "┌────────┬────────────────────────────┐\n", + "│ count │ url_host_registered_domain │\n", + "│ int64 │ varchar │\n", + "├────────┼────────────────────────────┤\n", + "│ 166381 │ yandex.ru │\n", + "│ 28067 │ zr.ru │\n", + "│ 23968 │ zin.ru │\n", + "│ 18272 │ zarplata.ru │\n", + "│ 16030 │ zab.ru │\n", + "│ 13283 │ yugzone.ru │\n", + "│ 12003 │ yapl.ru │\n", + "│ 10861 │ zp-news.ru │\n", + "│ 10348 │ zavod-metakon.ru │\n", + "│ 9794 │ zhilibyli.ru │\n", + "│ · │ · │\n", + "│ · │ · │\n", + "│ · │ · │\n", + "│ 100 │ yardeloros.ru │\n", + "│ 100 │ zov-vrn.ru │\n", + "│ 100 │ yukiandyan.ru │\n", + "│ 100 │ zapashnyh.ru │\n", + "│ 100 │ zakaztortikov.ru │\n", + "│ 100 │ zvuk4event.ru │\n", + "│ 100 │ zharim-myaso.ru │\n", + "│ 100 │ zelaomed.ru │\n", + "│ 100 │ zapchasti-bentley.ru │\n", + "│ 100 │ yoga-plus.ru │\n", + "└────────┴────────────────────────────┘\n", + " 3876 rows (20 shown) 2 columns\n", + "\n" + ] + } + ], + "source": [ + "duckdb.sql(f\"\"\"SELECT COUNT(*) AS count,\n", + " url_host_registered_domain\n", + "FROM read_parquet({http_index_files!r}, hive_partitioning = true) \n", + "WHERE crawl = 'CC-MAIN-2026-17' AND subset = 'warc'\n", + " AND url_host_tld = 'ru'\n", + "GROUP BY url_host_registered_domain\n", + "HAVING (COUNT(*) >= 100)\n", + "ORDER BY count DESC\"\"\").show()" + ] + }, + { + "cell_type": "markdown", + "id": "1d838c0f", + "metadata": {}, + "source": [ + "## Query 2: Finding Domains with Multilingual Content\n", + "\n", + "The next example demonstrates the power of Trino functions by looking for domains with multilingual content. It uses regular expressions to identify [ISO 639-1](https://www.iso.org/iso-639-language-code) language codes in the URL: e.g. the \"en\" in `https://example.com/about/en/page`.html.\n", + "\n", + "For demonstration purposes, the query is restricted to a single top-level domain (`.ru`). Run the following query:\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "bed6b687", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "┌────────────────────────┬────────┬─────────┬───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐\n", + "│ domain │ n_lang │ n_pages │ lang_counts │\n", + "│ varchar │ int64 │ int64 │ map(varchar, ubigint) │\n", + "├────────────────────────┼────────┼─────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤\n", + "│ yapl.ru │ 5 │ 4428 │ {ip=153, ix=6, rf=1028, ru=2948, su=293} │\n", + "│ zr.ru │ 10 │ 4267 │ {bg=5, ds=5, dy=1, es=1, gt=1, lx=1, mg=2, sp=5, ux=1, zr=4245} │\n", + "│ yandex.ru │ 68 │ 2212 │ {ac=1, ae=1, af=1, ap=86, as=1, bc=2, bf=1, ca=1, cb=1, cd=1, cf=1, cr=1, da=1, db=1, dd=1, de=1, dl=7, dm=1, ds=14, ea=1, eb=1, ed=5, ee=1, el=4, en=7, fa=1, fc=3, fe=1, fq=1, gd=2, gn=1, gv=2, ic=1, id=1914, ir=1, iw=1, ja=1, jd=1, jg=1, le=1, lp=1, lr=1, mc=1, mg=39, mr=1, nb=1, nc=2, nd=1, ng=1, nn=2, nq=1, ob=11, ok=1, os=29, qr=1, qt=1, re=2, rh=4, ru=20, sg=1, sv=1, tc=1, tk=2, vc=1, wb=1, xv=7, xz=2, yu=1} │\n", + "│ zelenypodokonnik.ru │ 1 │ 1548 │ {en=1548} │\n", + "│ znaemvseh.ru │ 2 │ 1036 │ {ip=438, ul=598} │\n", + "│ yeahub.ru │ 1 │ 987 │ {ru=987} │\n", + "│ zoroastrian.ru │ 1 │ 985 │ {en=985} │\n", + "│ zaryavladivostok.ru │ 2 │ 971 │ {en=463, ru=508} │\n", + "│ zdes-kursy.ru │ 2 │ 898 │ {pk=424, pp=474} │\n", + "│ zenstat.ru │ 1 │ 885 │ {id=885} │\n", + "│ · │ · │ · │ · │\n", + "│ · │ · │ · │ · │\n", + "│ · │ · │ · │ · │\n", + "│ z-zhizn.ru │ 1 │ 134 │ {ru=134} │\n", + "│ yappix.ru │ 2 │ 131 │ {en=128, eu=3} │\n", + "│ zazakon.ru │ 1 │ 128 │ {gb=128} │\n", + "│ ziranov.ru │ 1 │ 126 │ {up=126} │\n", + "│ ykoctpa.ru │ 1 │ 125 │ {my=125} │\n", + "│ yukon-repair-center.ru │ 2 │ 115 │ {rt=65, xt=50} │\n", + "│ yesreplace.ru │ 1 │ 114 │ {nn=114} │\n", + "│ youvegan.ru │ 4 │ 114 │ {en=26, es=40, fr=23, pt=25} │\n", + "│ yoauto.ru │ 4 │ 105 │ {en=31, es=23, fr=25, pt=26} │\n", + "│ zharar.ru │ 1 │ 103 │ {kz=103} │\n", + "└────────────────────────┴────────┴─────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n", + " 51 rows (20 shown) 4 columns\n", + "\n" + ] + } + ], + "source": [ + "duckdb.query(f\"\"\"SELECT url_host_registered_domain AS domain,\n", + " COUNT(DISTINCT(url_path_lang)) as n_lang,\n", + " COUNT(*) as n_pages,\n", + " histogram(url_path_lang) as lang_counts\n", + "FROM read_parquet({http_index_files!r}, hive_partitioning = true),\n", + " UNNEST(regexp_extract_all(url_path, '/([a-z][a-z])/', 1)) AS t (url_path_lang)\n", + "WHERE crawl = 'CC-MAIN-2026-17'\n", + " AND subset = 'warc'\n", + " AND url_host_registry_suffix = 'ru'\n", + "GROUP BY url_host_registered_domain\n", + "HAVING COUNT(*) >= 100\n", + " AND COUNT(DISTINCT(url_path_lang)) >= 1\n", + "ORDER BY n_pages DESC\"\"\").show()" + ] + }, + { + "cell_type": "markdown", + "id": "e95d3469", + "metadata": {}, + "source": [ + "The result should contain four columns:\n", + "\n", + "- domain: (e.g. yandex.ru)\n", + "- n_lang: (a count of distinct languages detected in each domain)\n", + "- n_pages: (a count of pages in each domain)\n", + "- lang_counts: (a hash map of language codes and their counts, e.g. {de=2698, hi=1, ru=13, ...})\n", + "\n", + "The magic is done by the UNNEST command, which first extracts all two-letter path elements (e.g., `/en/`), then unrolls the elements into a new column `url_path_lang`. If two or more path elements are found, it creates multiple rows. Finally, it counts pages and unique languages and generates a histogram of language codes." + ] + }, + { + "cell_type": "markdown", + "id": "9b8550b2", + "metadata": {}, + "source": [ + "## See also\n", + "\n", + "Further information can be found at the documentation of the URL index:\n", + "\n", + "https://commoncrawl.org/url-index\n" + ] + }, + { + "cell_type": "markdown", + "id": "7331e962", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "cc-huggingface (3.12.13.final.0)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cc-huggingface/cdxt-hf.ipynb b/cc-huggingface/cdxt-hf.ipynb new file mode 100644 index 0000000..df0131e --- /dev/null +++ b/cc-huggingface/cdxt-hf.ipynb @@ -0,0 +1,37 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "b3466fdd", + "metadata": {}, + "source": [ + "# Using CDX toolkit via Hugging Face\n", + "\n", + "The [CDX toolkit](https://github.com/commoncrawl/cdx_toolkit) provides are accessing the Common Crawl index (and other indices such as the Inter Archive). In this notebook, we should how CDX toolkit can be used for querying the Common Crawl index hosted on Hugging Face.\n", + "\n", + "The Common Crawl's CDX index can be found here: https://huggingface.co/buckets/commoncrawl/commoncrawl\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b8b08d0", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "# TODO HF bucket holds currently only the CDX files" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cc-huggingface/requirements.txt b/cc-huggingface/requirements.txt new file mode 100644 index 0000000..35c4a8b --- /dev/null +++ b/cc-huggingface/requirements.txt @@ -0,0 +1,3 @@ +warcio[all] +huggingface_hub +jupyter diff --git a/cc-huggingface/s3-hf.ipynb b/cc-huggingface/s3-hf.ipynb new file mode 100644 index 0000000..32dac63 --- /dev/null +++ b/cc-huggingface/s3-hf.ipynb @@ -0,0 +1,166 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "31e4ac9e", + "metadata": {}, + "source": [ + "# Using Common Crawl data via Hugging Face's S3-compatible gateway\n", + "\n", + "The Common Crawl data stored on [the Hugging Face bucket](https://huggingface.co/buckets/commoncrawl/commoncrawl) should be usually accessed with the HF-native tools and API (see other notebooks). However, Hugging Face also provides a [S3-compatible gateway for its storage buckets](https://huggingface.co/docs/hub/storage-buckets-s3), i.e., if you already use S3 this is a way how you can reuse your existing tooling." + ] + }, + { + "cell_type": "markdown", + "id": "fcf90c83", + "metadata": {}, + "source": [ + "## Generating S3 Credentials\n", + "\n", + "Follow the instructions in the [official documentation](https://huggingface.co/docs/hub/storage-buckets-s3#generating-s3-credentials) to generate S3 credentials for your Hugging Face account. The credentials will similar to those below and should be added to your `~/.aws/credentials` together with an updated AWS profile:\n", + "\n", + "### Example: AWS CLI profile\n", + "\n", + "Add a profile to `~/.aws/config`:\n", + "\n", + "```\n", + "[profile hf]\n", + "region = us-east-1\n", + "endpoint_url = https://s3.hf.co/commoncrawl\n", + "s3 =\n", + " addressing_style = path\n", + " multipart_threshold = 2GB\n", + " multipart_chunksize = 2GB\n", + "request_checksum_calculation = when_required\n", + "response_checksum_validation = when_required\n", + "```\n", + "\n", + "Note: The official documentation uses `` placeholder. The placeholder needs to be replaced with `commoncrawl`.\n", + "\n", + "Add the matching credentials for the profile to `~/.aws/credentials`:\n", + "\n", + "```\n", + "[hf]\n", + "aws_access_key_id = HFAK...\n", + "aws_secret_access_key = ...\n", + "```\n", + "\n", + "## Accessing CC HF bucket with AWS CLI\n", + "\n", + "Now you can use the AWS S3 to access the Common Crawl HF bucket by using the `--profile hf` argument: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3d6ab2d7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " PRE CC-MAIN-2026-17/\n", + " PRE CC-MAIN-2026-21/\n", + " PRE CC-MAIN-2026-25/\n" + ] + } + ], + "source": [ + "# what crawl data is on HF?\n", + "!aws --profile hf s3 ls s3://commoncrawl/crawl-data/" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "4555697b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Total Objects: 2700\n", + " Total Size: 590.7 GiB\n" + ] + } + ], + "source": [ + "# how large is the index data ?\n", + "!aws --profile hf s3 ls s3://commoncrawl/cc-index/ --recursive --human-readable --summarize | tail -n 3" + ] + }, + { + "cell_type": "markdown", + "id": "1aa2f5a6", + "metadata": {}, + "source": [ + "## DuckDB via S3-gateway\n", + "\n", + "To use duckdb for index queries, you need to setup the S3 credetials as follows:\n", + "\n", + "```sql\n", + "-- 1. Create the S3-compatible secret\n", + "CREATE OR REPLACE SECRET secret (\n", + " TYPE s3,\n", + " PROVIDER config,\n", + " KEY_ID 'xxx',\n", + " SECRET 'yyy',\n", + " REGION 'us-east-1',\n", + " ENDPOINT 's3.hf.co/commoncrawl',\n", + " URL_STYLE 'path'\n", + ");\n", + "```\n", + "\n", + "And then you can use the same queries as you would with AWS S3 but via HF:\n", + "\n", + "```sql\n", + "SELECT * FROM 's3://commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-17/subset=crawldiagnostics/part-00000-ee7620e8-27a7-4c74-846b-a92b039dae92.c000.gz.parquet' \n", + "LIMIT 5;\n", + "```\n", + "\n", + "```\n", + "┌────────────────────────────────┬──────────────────────┬──────────────────────┬───┬──────────────────┬─────────────────┬──────────────────┐\n", + "│ url_surtkey │ url │ url_host_name │ … │ warc_segment │ crawl │ subset │\n", + "│ varchar │ varchar │ varchar │ … │ varchar │ varchar │ varchar │\n", + "├────────────────────────────────┼──────────────────────┼──────────────────────┼───┼──────────────────┼─────────────────┼──────────────────┤\n", + "│ ru,yandex,market-delivery)/zo… │ https://market-deli… │ market-delivery.yan… │ … │ 1775805909101.46 │ CC-MAIN-2026-17 │ crawldiagnostics │\n", + "│ ru,yandex,market-delivery)/zo… │ https://market-deli… │ market-delivery.yan… │ … │ 1775806209748.93 │ CC-MAIN-2026-17 │ crawldiagnostics │\n", + "│ ru,yandex,market-delivery)/zo… │ https://market-deli… │ market-delivery.yan… │ … │ 1775805908721.20 │ CC-MAIN-2026-17 │ crawldiagnostics │\n", + "│ ru,yandex,market-delivery)/zo… │ https://market-deli… │ market-delivery.yan… │ … │ 1775805908780.58 │ CC-MAIN-2026-17 │ crawldiagnostics │\n", + "│ ru,yandex,market-delivery)/zo… │ https://market-deli… │ market-delivery.yan… │ … │ 1775805908654.63 │ CC-MAIN-2026-17 │ crawldiagnostics │\n", + "└────────────────────────────────┴──────────────────────┴──────────────────────┴───┴──────────────────┴─────────────────┴──────────────────┘\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "be19315c", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "cc-huggingface (3.12.13.final.0)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cc-huggingface/warcio-hf.ipynb b/cc-huggingface/warcio-hf.ipynb new file mode 100644 index 0000000..2dd8434 --- /dev/null +++ b/cc-huggingface/warcio-hf.ipynb @@ -0,0 +1,149 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "999395ef", + "metadata": {}, + "source": [ + "# Reading WARC files from Hugging Face\n", + "\n", + "The Common Crawl HF bucket is available at: https://huggingface.co/buckets/commoncrawl/commoncrawl\n", + "\n", + "Hugging Face providers a file system like access to models, datasets, and buckets via the `hf://` protocol, i.e., a [pythonic fsspec-compatible file interface to the Hugging Face Hub](https://huggingface.co/docs/huggingface_hub/main/en/guides/hf_file_system). The WARC reader/writer tool `warcio` fully supports [fsspec](https://filesystem-spec.readthedocs.io/en/latest/). Make sure to install the optional `warcio` dependencies and the `huggingface_hub` package." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "65ada1e0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "warcinfo\n", + "request\n", + "response\n", + "http://003ms.ru/catalog/lekarstvennye-sredstva/nervnaya-sistema/antigrippin-312/tabletki-250-mgplus3-mgplus50-mg-dlya-detej-30-shtuk-shipuchie\n", + "metadata\n", + "request\n", + "response\n", + "http://029nfk.com/gjjb/ml/9043.html\n", + "metadata\n", + "request\n", + "response\n", + "http://0401tt.com/?PUT=a_show&AID=241820&FID=133150&R2=&CHANNEL=\n", + "metadata\n", + "request\n" + ] + } + ], + "source": [ + "# Read a WARC file via fsspec and hf:// protocol\n", + "import fsspec\n", + "from warcio.archiveiterator import ArchiveIterator\n", + "\n", + "with fsspec.open('hf://buckets/commoncrawl/commoncrawl/crawl-data/CC-MAIN-2026-17/segments/1775805908305.14/warc/CC-MAIN-20260410081153-20260410111153-00000.warc.gz', 'rb') as stream:\n", + " for i, record in enumerate(ArchiveIterator(stream)):\n", + " print(record.rec_type)\n", + "\n", + " if record.rec_type == 'response':\n", + " print(record.rec_headers.get_header('WARC-Target-URI'))\n", + "\n", + " if i >= 10:\n", + " break" + ] + }, + { + "cell_type": "markdown", + "id": "edf44401", + "metadata": {}, + "source": [ + "The `warcio` CLI can directly read from the HF bucket:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "c58c8cda", + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"offset\": \"0\", \"content-type\": \"application/warc-fields\"}\n", + "{\"offset\": \"470\", \"content-type\": \"application/http; msgtype=request\", \"warc-target-uri\": \"http://003ms.ru/catalog/lekarstvennye-sredstva/nervnaya-sistema/antigrippin-312/tabletki-250-mgplus3-mgplus50-mg-dlya-detej-30-shtuk-shipuchie\"}\n", + "{\"offset\": \"984\", \"content-type\": \"application/http; msgtype=response\", \"http:content-type\": \"text/html; charset=UTF-8\", \"warc-target-uri\": \"http://003ms.ru/catalog/lekarstvennye-sredstva/nervnaya-sistema/antigrippin-312/tabletki-250-mgplus3-mgplus50-mg-dlya-detej-30-shtuk-shipuchie\"}\n", + "{\"offset\": \"29505\", \"content-type\": \"application/warc-fields\", \"warc-target-uri\": \"http://003ms.ru/catalog/lekarstvennye-sredstva/nervnaya-sistema/antigrippin-312/tabletki-250-mgplus3-mgplus50-mg-dlya-detej-30-shtuk-shipuchie\"}\n", + "{\"offset\": \"29997\", \"content-type\": \"application/http; msgtype=request\", \"warc-target-uri\": \"http://029nfk.com/gjjb/ml/9043.html\"}\n", + "{\"offset\": \"30438\", \"content-type\": \"application/http; msgtype=response\", \"http:content-type\": \"text/html\", \"warc-target-uri\": \"http://029nfk.com/gjjb/ml/9043.html\"}\n", + "{\"offset\": \"35564\", \"content-type\": \"application/warc-fields\", \"warc-target-uri\": \"http://029nfk.com/gjjb/ml/9043.html\"}\n", + "{\"offset\": \"35988\", \"content-type\": \"application/http; msgtype=request\", \"warc-target-uri\": \"http://0401tt.com/?PUT=a_show&AID=241820&FID=133150&R2=&CHANNEL=\"}\n", + "{\"offset\": \"36497\", \"content-type\": \"application/http; msgtype=response\", \"http:content-type\": \"text/html; charset=Big5\", \"warc-target-uri\": \"http://0401tt.com/?PUT=a_show&AID=241820&FID=133150&R2=&CHANNEL=\"}\n", + "{\"offset\": \"45158\", \"content-type\": \"application/warc-fields\", \"warc-target-uri\": \"http://0401tt.com/?PUT=a_show&AID=241820&FID=133150&R2=&CHANNEL=\"}\n", + "Traceback (most recent call last):\n", + " File \"/Users/moc-malteos/repos/commoncrawl/cc-notebooks/cc-huggingface/.venv/bin/warcio\", line 10, in \n", + " sys.exit(main())\n", + " ^^^^^^\n", + " File \"/Users/moc-malteos/repos/commoncrawl/cc-notebooks/cc-huggingface/.venv/lib/python3.12/site-packages/warcio/cli.py\", line 62, in main\n", + " cmd.func(cmd)\n", + " File \"/Users/moc-malteos/repos/commoncrawl/cc-notebooks/cc-huggingface/.venv/lib/python3.12/site-packages/warcio/cli.py\", line 74, in indexer\n", + " _indexer.process_all()\n", + " File \"/Users/moc-malteos/repos/commoncrawl/cc-notebooks/cc-huggingface/.venv/lib/python3.12/site-packages/warcio/indexer.py\", line 33, in process_all\n", + " self.process_one(fh, out, filename)\n", + " File \"/Users/moc-malteos/repos/commoncrawl/cc-notebooks/cc-huggingface/.venv/lib/python3.12/site-packages/warcio/indexer.py\", line 41, in process_one\n", + " self.process_index_entry(it, record, filename, output)\n", + " File \"/Users/moc-malteos/repos/commoncrawl/cc-notebooks/cc-huggingface/.venv/lib/python3.12/site-packages/warcio/indexer.py\", line 53, in process_index_entry\n", + " self._write_line(output, index, record, filename)\n", + " File \"/Users/moc-malteos/repos/commoncrawl/cc-notebooks/cc-huggingface/.venv/lib/python3.12/site-packages/warcio/indexer.py\", line 87, in _write_line\n", + " out.write(json.dumps(index) + '\\n')\n", + "BrokenPipeError: [Errno 32] Broken pipe\n" + ] + } + ], + "source": [ + "# Note: This command will trigger a broken pipe error after 10 records due to `head -n 10`\n", + "!warcio index hf://buckets/commoncrawl/commoncrawl/crawl-data/CC-MAIN-2026-17/segments/1775805908305.14/warc/CC-MAIN-20260410081153-20260410111153-00000.warc.gz -f offset,content-type,http:content-type,warc-target-uri | head -n 10" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25c0414b", + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "cc-huggingface (3.12.13.final.0)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}