From fbf95f63c4130ef60a577f249adf5550e5d6ceb3 Mon Sep 17 00:00:00 2001 From: Moshiur Rahman Moshi Date: Fri, 10 May 2024 04:58:32 -0500 Subject: [PATCH 1/4] Created using Colab --- wiki_dump_extractor.ipynb | 118 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 wiki_dump_extractor.ipynb diff --git a/wiki_dump_extractor.ipynb b/wiki_dump_extractor.ipynb new file mode 100644 index 00000000..50f01d0e --- /dev/null +++ b/wiki_dump_extractor.ipynb @@ -0,0 +1,118 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "authorship_tag": "ABX9TyOfBosmBwWK4vEUpPfFQjsR", + "include_colab_link": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "AIN1eV12yzVv", + "outputId": "d8543725-1023-4b42-8392-4434fe32adff" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting wikiextractor\n", + " Downloading wikiextractor-3.0.6-py3-none-any.whl (46 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.4/46.4 kB\u001b[0m \u001b[31m478.1 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: wikiextractor\n", + "Successfully installed wikiextractor-3.0.6\n" + ] + } + ], + "source": [ + "pip install wikiextractor" + ] + }, + { + "cell_type": "code", + "source": [ + "!wget https://dumps.wikimedia.org/bnwiki/latest/bnwiki-latest-pages-articles.xml.bz2\n", + "!bzip2 -d bnwiki-latest-pages-articles.xml.bz2\n", + "!python -m wikiextractor.WikiExtractor bnwiki-latest-pages-articles.xml" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "G1HIxA3u17o0", + "outputId": "c2997dd8-f849-4be6-83c1-0862f41b0fee" + }, + "execution_count": 31, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "--2024-05-10 09:31:22-- https://dumps.wikimedia.org/bnwiki/latest/bnwiki-latest-pages-articles.xml.bz2\n", + "Resolving dumps.wikimedia.org (dumps.wikimedia.org)... 208.80.154.71, 2620:0:861:3:208:80:154:71\n", + "Connecting to dumps.wikimedia.org (dumps.wikimedia.org)|208.80.154.71|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 380404074 (363M) [application/octet-stream]\n", + "Saving to: ‘bnwiki-latest-pages-articles.xml.bz2’\n", + "\n", + "bnwiki-latest-pages 100%[===================>] 362.78M 4.84MB/s in 77s \n", + "\n", + "2024-05-10 09:32:39 (4.71 MB/s) - ‘bnwiki-latest-pages-articles.xml.bz2’ saved [380404074/380404074]\n", + "\n", + "INFO: Preprocessing 'bnwiki-latest-pages-articles.xml' to collect template definitions: this may take some time.\n", + "INFO: Preprocessed 100000 pages\n", + "INFO: Preprocessed 200000 pages\n", + "INFO: Preprocessed 300000 pages\n", + "INFO: Preprocessed 400000 pages\n", + "INFO: Preprocessed 500000 pages\n", + "INFO: Preprocessed 600000 pages\n", + "INFO: Preprocessed 700000 pages\n", + "INFO: Loaded 166186 templates in 54.8s\n", + "INFO: Starting page extraction from bnwiki-latest-pages-articles.xml.\n", + "INFO: Using 1 extract processes.\n", + "INFO: Extracted 100000 articles (1218.9 art/s)\n", + "INFO: Extracted 200000 articles (939.3 art/s)\n", + "INFO: Extracted 300000 articles (741.1 art/s)\n", + "INFO: Finished 1-process extraction of 383474 articles in 459.8s (833.9 art/s)\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "!tar -cjf archive.tar.bz2 text\n" + ], + "metadata": { + "id": "IG0pMSjBzBQI" + }, + "execution_count": 40, + "outputs": [] + } + ] +} \ No newline at end of file From 558ecac9bf1ffc259e09d7e8cfc8605bc29ff237 Mon Sep 17 00:00:00 2001 From: Moshiur Rahman Moshi Date: Fri, 10 May 2024 04:58:47 -0500 Subject: [PATCH 2/4] Update README.md --- README.md | 193 +----------------------------------------------------- 1 file changed, 2 insertions(+), 191 deletions(-) diff --git a/README.md b/README.md index bd5407c9..7a6db853 100644 --- a/README.md +++ b/README.md @@ -1,196 +1,7 @@ # WikiExtractor [WikiExtractor.py](http://medialab.di.unipi.it/wiki/Wikipedia_Extractor) is a Python script that extracts and cleans text from a [Wikipedia database backup dump](https://dumps.wikimedia.org/), e.g. https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2 for English. -The tool is written in Python and requires Python 3 but no additional library. -**Warning**: problems have been reported on Windows due to poor support for `StringIO` in the Python implementation on Windows. +# Extract latest Wiki Article in colab -For further information, see the [Wiki](https://github.com/attardi/wikiextractor/wiki). +I have used this library and a simple colab uses. You can just add your desired language url and get your extracted articles: -# Wikipedia Cirrus Extractor - -`cirrus-extractor.py` is a version of the script that performs extraction from a Wikipedia Cirrus dump. -Cirrus dumps contain text with already expanded templates. - -Cirrus dumps are available at: -[cirrussearch](http://dumps.wikimedia.org/other/cirrussearch/). - -# Details - -WikiExtractor performs template expansion by preprocessing the whole dump and extracting template definitions. - -In order to speed up processing: - -- multiprocessing is used for dealing with articles in parallel -- a cache is kept of parsed templates (only useful for repeated extractions). - -## Installation - -The script may be invoked directly: - - python -m wikiextractor.WikiExtractor - -It can also be installed from `PyPi` by doing: - - pip install wikiextractor - -or locally with: - - (sudo) python setup.py install - -The installer also installs two scripts for direct invocation: - - wikiextractor (equivalent to python -m wikiextractor.WikiExtractor) - extractPage (to extract a single page from a dump) - -## Usage - -### Wikiextractor -The script is invoked with a Wikipedia dump file as an argument: - - python -m wikiextractor.WikiExtractor [--templates ] - -The option `--templates` extracts the templates to a local file, which can be reloaded to reduce the time to perform extraction. - -The output is stored in several files of similar size in a given directory. -Each file will contains several documents in this [document format](https://github.com/attardi/wikiextractor/wiki/File-Format). - -``` -usage: wikiextractor [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--json] [--html] [-l] [-ns ns1,ns2] - [--templates TEMPLATES] [--no-templates] [--html-safe HTML_SAFE] [--processes PROCESSES] - [-q] [--debug] [-a] [-v] - input - -Wikipedia Extractor: -Extracts and cleans text from a Wikipedia database dump and stores output in a -number of files of similar size in a given directory. -Each file will contain several documents in the format: - - - ... - - -If the program is invoked with the --json flag, then each file will -contain several documents formatted as json ojects, one per line, with -the following structure - - {"id": "", "revid": "", "url": "", "title": "", "text": "..."} - -The program performs template expansion by preprocesssng the whole dump and -collecting template definitions. - -positional arguments: - input XML wiki dump file - -optional arguments: - -h, --help show this help message and exit - --processes PROCESSES - Number of processes to use (default 79) - -Output: - -o OUTPUT, --output OUTPUT - directory for extracted files (or '-' for dumping to stdout) - -b n[KMG], --bytes n[KMG] - maximum bytes per output file (default 1M) - -c, --compress compress output files using bzip - --json write output in json format instead of the default format - -Processing: - --html produce HTML output, subsumes --links - -l, --links preserve links - -ns ns1,ns2, --namespaces ns1,ns2 - accepted namespaces - --templates TEMPLATES - use or create file containing templates - --no-templates Do not expand templates - --html-safe HTML_SAFE - use to produce HTML safe output within ... - -Special: - -q, --quiet suppress reporting progress info - --debug print debug info - -a, --article analyze a file containing a single article (debug option) - -v, --version print program version -``` - -Saving templates to a file will speed up performing extraction the next time, -assuming template definitions have not changed. - -Option `--no-templates` significantly speeds up the extractor, avoiding the cost -of expanding [MediaWiki templates](https://www.mediawiki.org/wiki/Help:Templates). - -For further information, visit [the documentation](http://attardi.github.io/wikiextractor). - -### Cirrus Extractor - -~~~ -usage: cirrus-extract.py [-h] [-o OUTPUT] [-b n[KMG]] [-c] [-ns ns1,ns2] [-q] - [-v] - input - -Wikipedia Cirrus Extractor: -Extracts and cleans text from a Wikipedia Cirrus dump and stores output in a -number of files of similar size in a given directory. -Each file will contain several documents in the format: - - - ... - - -positional arguments: - input Cirrus Json wiki dump file - -optional arguments: - -h, --help show this help message and exit - -Output: - -o OUTPUT, --output OUTPUT - directory for extracted files (or '-' for dumping to - stdin) - -b n[KMG], --bytes n[KMG] - maximum bytes per output file (default 1M) - -c, --compress compress output files using bzip - -Processing: - -ns ns1,ns2, --namespaces ns1,ns2 - accepted namespaces - -Special: - -q, --quiet suppress reporting progress info - -v, --version print program version -~~~ - -### extractPage -Extract a single page from a Wikipedia dump file. - -~~~ -usage: extractPage [-h] [--id ID] [--template] [-v] input - -Wikipedia Page Extractor: -Extracts a single page from a Wikipedia dump file. - -positional arguments: - input XML wiki dump file - -optional arguments: - -h, --help show this help message and exit - --id ID article number - --template template number - -v, --version print program version -~~~ - -## License -The code is made available under the [GNU Affero General Public License v3.0](LICENSE). - -## Reference -If you find this code useful, please refer it in publications as: - -~~~ -@misc{Wikiextractor2015, - author = {Giusepppe Attardi}, - title = {WikiExtractor}, - year = {2015}, - publisher = {GitHub}, - journal = {GitHub repository}, - howpublished = {\url{https://github.com/attardi/wikiextractor}} -} -~~~ From 521d2814f5230423ecee020dba4ca06563d3db28 Mon Sep 17 00:00:00 2001 From: moshi Date: Sat, 11 May 2024 00:22:43 -0500 Subject: [PATCH 3/4] wikiextractor used in ipynb file tested in google colab --- wiki_dump_extractor.ipynb | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/wiki_dump_extractor.ipynb b/wiki_dump_extractor.ipynb index 50f01d0e..087ebbda 100644 --- a/wiki_dump_extractor.ipynb +++ b/wiki_dump_extractor.ipynb @@ -3,9 +3,7 @@ "nbformat_minor": 0, "metadata": { "colab": { - "provenance": [], - "authorship_tag": "ABX9TyOfBosmBwWK4vEUpPfFQjsR", - "include_colab_link": true + "provenance": [] }, "kernelspec": { "name": "python3", @@ -16,16 +14,6 @@ } }, "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "view-in-github", - "colab_type": "text" - }, - "source": [ - "\"Open" - ] - }, { "cell_type": "code", "execution_count": 1, From a629d00a4b539e81039ebee9ba3219905ab4e866 Mon Sep 17 00:00:00 2001 From: moshi Date: Sat, 11 May 2024 00:34:28 -0500 Subject: [PATCH 4/4] readme reset to original --- README.md | 193 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 191 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7a6db853..bd5407c9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,196 @@ # WikiExtractor [WikiExtractor.py](http://medialab.di.unipi.it/wiki/Wikipedia_Extractor) is a Python script that extracts and cleans text from a [Wikipedia database backup dump](https://dumps.wikimedia.org/), e.g. https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2 for English. -# Extract latest Wiki Article in colab +The tool is written in Python and requires Python 3 but no additional library. +**Warning**: problems have been reported on Windows due to poor support for `StringIO` in the Python implementation on Windows. -I have used this library and a simple colab uses. You can just add your desired language url and get your extracted articles: +For further information, see the [Wiki](https://github.com/attardi/wikiextractor/wiki). +# Wikipedia Cirrus Extractor + +`cirrus-extractor.py` is a version of the script that performs extraction from a Wikipedia Cirrus dump. +Cirrus dumps contain text with already expanded templates. + +Cirrus dumps are available at: +[cirrussearch](http://dumps.wikimedia.org/other/cirrussearch/). + +# Details + +WikiExtractor performs template expansion by preprocessing the whole dump and extracting template definitions. + +In order to speed up processing: + +- multiprocessing is used for dealing with articles in parallel +- a cache is kept of parsed templates (only useful for repeated extractions). + +## Installation + +The script may be invoked directly: + + python -m wikiextractor.WikiExtractor + +It can also be installed from `PyPi` by doing: + + pip install wikiextractor + +or locally with: + + (sudo) python setup.py install + +The installer also installs two scripts for direct invocation: + + wikiextractor (equivalent to python -m wikiextractor.WikiExtractor) + extractPage (to extract a single page from a dump) + +## Usage + +### Wikiextractor +The script is invoked with a Wikipedia dump file as an argument: + + python -m wikiextractor.WikiExtractor [--templates ] + +The option `--templates` extracts the templates to a local file, which can be reloaded to reduce the time to perform extraction. + +The output is stored in several files of similar size in a given directory. +Each file will contains several documents in this [document format](https://github.com/attardi/wikiextractor/wiki/File-Format). + +``` +usage: wikiextractor [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--json] [--html] [-l] [-ns ns1,ns2] + [--templates TEMPLATES] [--no-templates] [--html-safe HTML_SAFE] [--processes PROCESSES] + [-q] [--debug] [-a] [-v] + input + +Wikipedia Extractor: +Extracts and cleans text from a Wikipedia database dump and stores output in a +number of files of similar size in a given directory. +Each file will contain several documents in the format: + + + ... + + +If the program is invoked with the --json flag, then each file will +contain several documents formatted as json ojects, one per line, with +the following structure + + {"id": "", "revid": "", "url": "", "title": "", "text": "..."} + +The program performs template expansion by preprocesssng the whole dump and +collecting template definitions. + +positional arguments: + input XML wiki dump file + +optional arguments: + -h, --help show this help message and exit + --processes PROCESSES + Number of processes to use (default 79) + +Output: + -o OUTPUT, --output OUTPUT + directory for extracted files (or '-' for dumping to stdout) + -b n[KMG], --bytes n[KMG] + maximum bytes per output file (default 1M) + -c, --compress compress output files using bzip + --json write output in json format instead of the default format + +Processing: + --html produce HTML output, subsumes --links + -l, --links preserve links + -ns ns1,ns2, --namespaces ns1,ns2 + accepted namespaces + --templates TEMPLATES + use or create file containing templates + --no-templates Do not expand templates + --html-safe HTML_SAFE + use to produce HTML safe output within ... + +Special: + -q, --quiet suppress reporting progress info + --debug print debug info + -a, --article analyze a file containing a single article (debug option) + -v, --version print program version +``` + +Saving templates to a file will speed up performing extraction the next time, +assuming template definitions have not changed. + +Option `--no-templates` significantly speeds up the extractor, avoiding the cost +of expanding [MediaWiki templates](https://www.mediawiki.org/wiki/Help:Templates). + +For further information, visit [the documentation](http://attardi.github.io/wikiextractor). + +### Cirrus Extractor + +~~~ +usage: cirrus-extract.py [-h] [-o OUTPUT] [-b n[KMG]] [-c] [-ns ns1,ns2] [-q] + [-v] + input + +Wikipedia Cirrus Extractor: +Extracts and cleans text from a Wikipedia Cirrus dump and stores output in a +number of files of similar size in a given directory. +Each file will contain several documents in the format: + + + ... + + +positional arguments: + input Cirrus Json wiki dump file + +optional arguments: + -h, --help show this help message and exit + +Output: + -o OUTPUT, --output OUTPUT + directory for extracted files (or '-' for dumping to + stdin) + -b n[KMG], --bytes n[KMG] + maximum bytes per output file (default 1M) + -c, --compress compress output files using bzip + +Processing: + -ns ns1,ns2, --namespaces ns1,ns2 + accepted namespaces + +Special: + -q, --quiet suppress reporting progress info + -v, --version print program version +~~~ + +### extractPage +Extract a single page from a Wikipedia dump file. + +~~~ +usage: extractPage [-h] [--id ID] [--template] [-v] input + +Wikipedia Page Extractor: +Extracts a single page from a Wikipedia dump file. + +positional arguments: + input XML wiki dump file + +optional arguments: + -h, --help show this help message and exit + --id ID article number + --template template number + -v, --version print program version +~~~ + +## License +The code is made available under the [GNU Affero General Public License v3.0](LICENSE). + +## Reference +If you find this code useful, please refer it in publications as: + +~~~ +@misc{Wikiextractor2015, + author = {Giusepppe Attardi}, + title = {WikiExtractor}, + year = {2015}, + publisher = {GitHub}, + journal = {GitHub repository}, + howpublished = {\url{https://github.com/attardi/wikiextractor}} +} +~~~