Pridanie parsovacieho nastroja a notebooku pre konvertovanie parsovanych clankov
This commit is contained in:
parent
030dd1b4a1
commit
06ad8f769a
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
wikidump/
|
||||||
|
json/
|
||||||
|
.ipynb_checkpoints/
|
||||||
|
parsed.json
|
||||||
135
bz2tojson.ipynb
Normal file
135
bz2tojson.ipynb
Normal file
@ -0,0 +1,135 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"import pandas as pd\n",
|
||||||
|
"import re\n",
|
||||||
|
"from bs4 import BeautifulSoup\n",
|
||||||
|
"from glob import glob\n",
|
||||||
|
"import json\n",
|
||||||
|
"from tqdm import tqdm_notebook as tqdm"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 14,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"files = glob('D:\\\\Desktop\\\\diplomka\\\\wikidump\\\\parsed\\\\*')"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 17,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"application/vnd.jupyter.widget-view+json": {
|
||||||
|
"model_id": "83cff3cf94b54fb4a030c5c493c90ddc",
|
||||||
|
"version_major": 2,
|
||||||
|
"version_minor": 0
|
||||||
|
},
|
||||||
|
"text/plain": [
|
||||||
|
"HBox(children=(IntProgress(value=0, max=327), HTML(value='')))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "display_data"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"file D:\\Desktop\\diplomka\\wikidump\\parsed\\wiki_140 is has a formatting issue near <a> ), vysoký tón </a>\n",
|
||||||
|
"file D:\\Desktop\\diplomka\\wikidump\\parsed\\wiki_140 is has a formatting issue near <a> ).\n",
|
||||||
|
"\n",
|
||||||
|
"Podľa tónu a výslovnosti sa výrazne mení význam toho istého slova. Napríklad \"ike\" môže znamenať \"zadok\", \"silu\", \"rozdeliť\", \"spájať\". Slovo \"oke\" podobne môže znamenať \"diel\", \"hranica\", \"muž\" alebo \"potkan\".\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"</a>\n",
|
||||||
|
"file D:\\Desktop\\diplomka\\wikidump\\parsed\\wiki_190 is has a formatting issue near <a>\" vyhovuje predpokladom </a>\n",
|
||||||
|
"file D:\\Desktop\\diplomka\\wikidump\\parsed\\wiki_88 is has a formatting issue near <a ...=\"\" r=\"\">\n",
|
||||||
|
"\n",
|
||||||
|
"Dôležitým relačným systémom je </a>\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"parsed = []\n",
|
||||||
|
"\n",
|
||||||
|
"index = 0\n",
|
||||||
|
"for i in tqdm(range(len(files))):\n",
|
||||||
|
" f = open(files[i], \"r\", encoding=\"utf-8\")\n",
|
||||||
|
" content = f.read()\n",
|
||||||
|
" soup = BeautifulSoup(content)\n",
|
||||||
|
" \n",
|
||||||
|
" docs = soup.find_all('doc')\n",
|
||||||
|
" \n",
|
||||||
|
" for doc in docs:\n",
|
||||||
|
" id = doc['id']\n",
|
||||||
|
" title = doc['title']\n",
|
||||||
|
" url = doc['url']\n",
|
||||||
|
" paragraphs = doc.text.replace('\\n', '').replace('\\xa0', ' ').split('Section::::')\n",
|
||||||
|
" hrefs = doc.find_all('a')\n",
|
||||||
|
" \n",
|
||||||
|
" references = []\n",
|
||||||
|
" for href in hrefs:\n",
|
||||||
|
" try:\n",
|
||||||
|
" references.append(href['href'])\n",
|
||||||
|
" except KeyError:\n",
|
||||||
|
" print(f'file {files[i]} is has a formatting issue near {href}')\n",
|
||||||
|
" \n",
|
||||||
|
" parsed.append({\n",
|
||||||
|
" title:{\n",
|
||||||
|
" 'id': id,\n",
|
||||||
|
" 'url': url,\n",
|
||||||
|
" 'paragraphs': paragraphs, \n",
|
||||||
|
" 'references': list(set(references))\n",
|
||||||
|
" }\n",
|
||||||
|
" })\n",
|
||||||
|
" \n",
|
||||||
|
" with open(f'json/file{index}.json', 'w+') as fp:\n",
|
||||||
|
" json.dump(parsed, fp)\n",
|
||||||
|
" parsed = []\n",
|
||||||
|
" index += 1"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
||||||
93
wikiextractor/.gitignore
vendored
Normal file
93
wikiextractor/.gitignore
vendored
Normal file
@ -0,0 +1,93 @@
|
|||||||
|
local/
|
||||||
|
tmp/
|
||||||
|
|
||||||
|
### https://raw.github.com/github/gitignore/c699a4f4684e9e294c9c550f820ca330f019b6f9/python.gitignore
|
||||||
|
|
||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*$py.class
|
||||||
|
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
env/
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*,cover
|
||||||
|
.hypothesis/
|
||||||
|
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
local_settings.py
|
||||||
|
|
||||||
|
# Flask instance folder
|
||||||
|
instance/
|
||||||
|
|
||||||
|
# Scrapy stuff:
|
||||||
|
.scrapy
|
||||||
|
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
|
||||||
|
# PyBuilder
|
||||||
|
target/
|
||||||
|
|
||||||
|
# IPython Notebook
|
||||||
|
.ipynb_checkpoints
|
||||||
|
|
||||||
|
# pyenv
|
||||||
|
.python-version
|
||||||
|
|
||||||
|
# celery beat schedule file
|
||||||
|
celerybeat-schedule
|
||||||
|
|
||||||
|
# dotenv
|
||||||
|
.env
|
||||||
|
|
||||||
|
# virtualenv
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
|
||||||
|
# Spyder project settings
|
||||||
|
.spyderproject
|
||||||
|
|
||||||
|
# Editor files
|
||||||
|
*.idea
|
||||||
135
wikiextractor/README.md
Normal file
135
wikiextractor/README.md
Normal file
@ -0,0 +1,135 @@
|
|||||||
|
# WikiExtractor
|
||||||
|
[WikiExtractor.py](http://medialab.di.unipi.it/wiki/Wikipedia_Extractor) is a Python script that extracts and cleans text from a [Wikipedia database dump](http://download.wikimedia.org/).
|
||||||
|
|
||||||
|
The tool is written in Python and requires Python 2.7 or Python 3.3+ but no additional library.
|
||||||
|
|
||||||
|
For further information, see the [project Home Page](http://medialab.di.unipi.it/wiki/Wikipedia_Extractor) or the [Wiki](https://github.com/attardi/wikiextractor/wiki).
|
||||||
|
|
||||||
|
# Wikipedia Cirrus Extractor
|
||||||
|
|
||||||
|
`cirrus-extractor.py` is a version of the script that performs extraction from a Wikipedia Cirrus dump.
|
||||||
|
Cirrus dumps contain text with already expanded templates.
|
||||||
|
|
||||||
|
Cirrus dumps are available at:
|
||||||
|
[cirrussearch](http://dumps.wikimedia.org/other/cirrussearch/).
|
||||||
|
|
||||||
|
# Details
|
||||||
|
|
||||||
|
WikiExtractor performs template expansion by preprocessing the whole dump and extracting template definitions.
|
||||||
|
|
||||||
|
In order to speed up processing:
|
||||||
|
|
||||||
|
- multiprocessing is used for dealing with articles in parallel
|
||||||
|
- a cache is kept of parsed templates (only useful for repeated extractions).
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
The script may be invoked directly, however it can be installed by doing:
|
||||||
|
|
||||||
|
(sudo) python setup.py install
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
The script is invoked with a Wikipedia dump file as an argument.
|
||||||
|
The output is stored in several files of similar size in a given directory.
|
||||||
|
Each file will contains several documents in this [document format](http://medialab.di.unipi.it/wiki/Document_Format).
|
||||||
|
|
||||||
|
usage: WikiExtractor.py [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--json] [--html]
|
||||||
|
[-l] [-s] [--lists] [-ns ns1,ns2]
|
||||||
|
[--templates TEMPLATES] [--no-templates] [-r]
|
||||||
|
[--min_text_length MIN_TEXT_LENGTH]
|
||||||
|
[--filter_category path_of_categories_file]
|
||||||
|
[--filter_disambig_pages] [-it abbr,b,big]
|
||||||
|
[-de gallery,timeline,noinclude] [--keep_tables]
|
||||||
|
[--processes PROCESSES] [-q] [--debug] [-a] [-v]
|
||||||
|
[--log_file]
|
||||||
|
input
|
||||||
|
|
||||||
|
Wikipedia Extractor:
|
||||||
|
Extracts and cleans text from a Wikipedia database dump and stores output in a
|
||||||
|
number of files of similar size in a given directory.
|
||||||
|
Each file will contain several documents in the format:
|
||||||
|
|
||||||
|
<doc id="" revid="" url="" title="">
|
||||||
|
...
|
||||||
|
</doc>
|
||||||
|
|
||||||
|
If the program is invoked with the --json flag, then each file will
|
||||||
|
contain several documents formatted as json ojects, one per line, with
|
||||||
|
the following structure
|
||||||
|
|
||||||
|
{"id": "", "revid": "", "url":"", "title": "", "text": "..."}
|
||||||
|
|
||||||
|
Template expansion requires preprocesssng first the whole dump and
|
||||||
|
collecting template definitions.
|
||||||
|
|
||||||
|
positional arguments:
|
||||||
|
input XML wiki dump file
|
||||||
|
|
||||||
|
optional arguments:
|
||||||
|
-h, --help show this help message and exit
|
||||||
|
--processes PROCESSES
|
||||||
|
Number of processes to use (default 1)
|
||||||
|
|
||||||
|
Output:
|
||||||
|
-o OUTPUT, --output OUTPUT
|
||||||
|
directory for extracted files (or '-' for dumping to
|
||||||
|
stdout)
|
||||||
|
-b n[KMG], --bytes n[KMG]
|
||||||
|
maximum bytes per output file (default 1M)
|
||||||
|
-c, --compress compress output files using bzip
|
||||||
|
--json write output in json format instead of the default one
|
||||||
|
|
||||||
|
Processing:
|
||||||
|
--html produce HTML output, subsumes --links
|
||||||
|
-l, --links preserve links
|
||||||
|
-s, --sections preserve sections
|
||||||
|
--lists preserve lists
|
||||||
|
-ns ns1,ns2, --namespaces ns1,ns2
|
||||||
|
accepted namespaces in links
|
||||||
|
--templates TEMPLATES
|
||||||
|
use or create file containing templates
|
||||||
|
--no-templates Do not expand templates
|
||||||
|
-r, --revision Include the document revision id (default=False)
|
||||||
|
--min_text_length MIN_TEXT_LENGTH
|
||||||
|
Minimum expanded text length required to write
|
||||||
|
document (default=0)
|
||||||
|
--filter_category path_of_categories_file
|
||||||
|
Include or exclude specific categories from the dataset. Specify the categories in
|
||||||
|
file 'path_of_categories_file'. Format:
|
||||||
|
One category one line, and if the line starts with:
|
||||||
|
1) #: Comments, ignored;
|
||||||
|
2) ^: the categories will be in excluding-categories
|
||||||
|
3) others: the categories will be in including-categories.
|
||||||
|
Priority:
|
||||||
|
1) If excluding-categories is not empty, and any category of a page exists in excluding-categories, the page will be excluded; else
|
||||||
|
2) If including-categories is not empty, and no category of a page exists in including-categories, the page will be excluded; else
|
||||||
|
3) the page will be included
|
||||||
|
|
||||||
|
--filter_disambig_pages
|
||||||
|
Remove pages from output that contain disabmiguation
|
||||||
|
markup (default=False)
|
||||||
|
-it abbr,b,big, --ignored_tags abbr,b,big
|
||||||
|
comma separated list of tags that will be dropped,
|
||||||
|
keeping their content
|
||||||
|
-de gallery,timeline,noinclude, --discard_elements gallery,timeline,noinclude
|
||||||
|
comma separated list of elements that will be removed
|
||||||
|
from the article text
|
||||||
|
--keep_tables Preserve tables in the output article text
|
||||||
|
(default=False)
|
||||||
|
|
||||||
|
Special:
|
||||||
|
-q, --quiet suppress reporting progress info
|
||||||
|
--debug print debug info
|
||||||
|
-a, --article analyze a file containing a single article (debug
|
||||||
|
option)
|
||||||
|
-v, --version print program version
|
||||||
|
--log_file specify a file to save the log information.
|
||||||
|
|
||||||
|
|
||||||
|
Saving templates to a file will speed up performing extraction the next time,
|
||||||
|
assuming template definitions have not changed.
|
||||||
|
|
||||||
|
Option --no-templates significantly speeds up the extractor, avoiding the cost
|
||||||
|
of expanding [MediaWiki templates](https://www.mediawiki.org/wiki/Help:Templates).
|
||||||
|
|
||||||
|
For further information, visit [the documentation](http://attardi.github.io/wikiextractor).
|
||||||
3297
wikiextractor/WikiExtractor.py
Normal file
3297
wikiextractor/WikiExtractor.py
Normal file
File diff suppressed because it is too large
Load Diff
36512
wikiextractor/categories.filter
Normal file
36512
wikiextractor/categories.filter
Normal file
File diff suppressed because it is too large
Load Diff
248
wikiextractor/cirrus-extract.py
Normal file
248
wikiextractor/cirrus-extract.py
Normal file
@ -0,0 +1,248 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
#
|
||||||
|
# =============================================================================
|
||||||
|
# Version: 1.00 (December 15, 2015)
|
||||||
|
# Author: Giuseppe Attardi (attardi@di.unipi.it), University of Pisa
|
||||||
|
#
|
||||||
|
# =============================================================================
|
||||||
|
# Copyright (c) 2015. Giuseppe Attardi (attardi@di.unipi.it).
|
||||||
|
# =============================================================================
|
||||||
|
# This file is part of Tanl.
|
||||||
|
#
|
||||||
|
# Tanl is free software; you can redistribute it and/or modify it
|
||||||
|
# under the terms of the GNU General Public License, version 3,
|
||||||
|
# as published by the Free Software Foundation.
|
||||||
|
#
|
||||||
|
# Tanl is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
"""Wikipedia Cirrus Extractor:
|
||||||
|
Extracts and cleans text from a Wikipedia Cirrus dump and stores output in a
|
||||||
|
number of files of similar size in a given directory.
|
||||||
|
Each file will contain several documents in the format:
|
||||||
|
|
||||||
|
<doc id="" url="" title="" language="" revision="">
|
||||||
|
...
|
||||||
|
</doc>
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys, os.path, time
|
||||||
|
import re
|
||||||
|
import json
|
||||||
|
import argparse
|
||||||
|
import bz2
|
||||||
|
import gzip
|
||||||
|
import logging
|
||||||
|
|
||||||
|
# Program version
|
||||||
|
version = '1.00'
|
||||||
|
|
||||||
|
urlbase = 'http://it.wikipedia.org/'
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
|
||||||
|
class NextFile(object):
|
||||||
|
"""
|
||||||
|
Synchronous generation of next available file name.
|
||||||
|
"""
|
||||||
|
|
||||||
|
filesPerDir = 100
|
||||||
|
|
||||||
|
def __init__(self, path_name):
|
||||||
|
self.path_name = path_name
|
||||||
|
self.dir_index = -1
|
||||||
|
self.file_index = -1
|
||||||
|
|
||||||
|
def next(self):
|
||||||
|
self.file_index = (self.file_index + 1) % NextFile.filesPerDir
|
||||||
|
if self.file_index == 0:
|
||||||
|
self.dir_index += 1
|
||||||
|
dirname = self._dirname()
|
||||||
|
if not os.path.isdir(dirname):
|
||||||
|
os.makedirs(dirname)
|
||||||
|
return self._filepath()
|
||||||
|
|
||||||
|
def _dirname(self):
|
||||||
|
char1 = self.dir_index % 26
|
||||||
|
char2 = self.dir_index / 26 % 26
|
||||||
|
return os.path.join(self.path_name, '%c%c' % (ord('A') + char2, ord('A') + char1))
|
||||||
|
|
||||||
|
def _filepath(self):
|
||||||
|
return '%s/wiki_%02d' % (self._dirname(), self.file_index)
|
||||||
|
|
||||||
|
class OutputSplitter(object):
|
||||||
|
"""
|
||||||
|
File-like object, that splits output to multiple files of a given max size.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, nextFile, max_file_size=0, compress=True):
|
||||||
|
"""
|
||||||
|
:param nextfile: a NextFile object from which to obtain filenames
|
||||||
|
to use.
|
||||||
|
:param max_file_size: the maximum size of each file.
|
||||||
|
:para compress: whether to write data with bzip compression.
|
||||||
|
"""
|
||||||
|
self.nextFile = nextFile
|
||||||
|
self.compress = compress
|
||||||
|
self.max_file_size = max_file_size
|
||||||
|
self.file = self.open(self.nextFile.next())
|
||||||
|
|
||||||
|
def reserve(self, size):
|
||||||
|
if self.file.tell() + size > self.max_file_size:
|
||||||
|
self.close()
|
||||||
|
self.file = self.open(self.nextFile.next())
|
||||||
|
|
||||||
|
def write(self, data):
|
||||||
|
self.reserve(len(data))
|
||||||
|
self.file.write(data)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.file.close()
|
||||||
|
|
||||||
|
def open(self, filename):
|
||||||
|
if self.compress:
|
||||||
|
return bz2.BZ2File(filename + '.bz2', 'w')
|
||||||
|
else:
|
||||||
|
return open(filename, 'w')
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
|
||||||
|
class Extractor(object):
|
||||||
|
|
||||||
|
def extract(self, out):
|
||||||
|
"""
|
||||||
|
:param out: output file.
|
||||||
|
"""
|
||||||
|
logging.debug("%s\t%s", self.id, self.title)
|
||||||
|
text = ''.join(self.page)
|
||||||
|
url = get_url(self.id)
|
||||||
|
header = '<doc id="%s" url="%s" title="%s" language="%s" revision="%s">\n' % (self.id, url, self.title, self.language, self.revision)
|
||||||
|
# Separate header from text with a newline.
|
||||||
|
header += self.title + '\n\n'
|
||||||
|
header = header.encode('utf-8')
|
||||||
|
footer = "\n</doc>\n"
|
||||||
|
out.write(header)
|
||||||
|
text = clean(self, text)
|
||||||
|
for line in compact(text):
|
||||||
|
out.write(line.encode('utf-8'))
|
||||||
|
out.write('\n')
|
||||||
|
out.write(footer)
|
||||||
|
|
||||||
|
def process_dump(input_file, out_file, file_size, file_compress):
|
||||||
|
"""
|
||||||
|
:param input_file: name of the wikipedia dump file; '-' to read from stdin
|
||||||
|
:param out_file: directory where to store extracted data, or '-' for stdout
|
||||||
|
:param file_size: max size of each extracted file, or None for no max (one file)
|
||||||
|
:param file_compress: whether to compress files with bzip.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if input_file == '-':
|
||||||
|
input = sys.stdin
|
||||||
|
else:
|
||||||
|
input = gzip.open(input_file)
|
||||||
|
|
||||||
|
if out_file == '-':
|
||||||
|
output = sys.stdout
|
||||||
|
if file_compress:
|
||||||
|
logging.warn("writing to stdout, so no output compression (use external tool)")
|
||||||
|
else:
|
||||||
|
nextFile = NextFile(out_file)
|
||||||
|
output = OutputSplitter(nextFile, file_size, file_compress)
|
||||||
|
|
||||||
|
# process dump
|
||||||
|
# format
|
||||||
|
# {"index":{"_type":"page","_id":"3825914"}}
|
||||||
|
# {"namespace":0,"title":TITLE,"timestamp":"2014-06-29T15:51:09Z","text":TEXT,...}
|
||||||
|
while True:
|
||||||
|
line = input.readline()
|
||||||
|
if not line:
|
||||||
|
break
|
||||||
|
index = json.loads(line)
|
||||||
|
content = json.loads(input.readline())
|
||||||
|
type = index['index']['_type']
|
||||||
|
id = index['index']['_id']
|
||||||
|
language = content['language']
|
||||||
|
revision = content['version']
|
||||||
|
if type == 'page' and content['namespace'] == 0:
|
||||||
|
title = content['title']
|
||||||
|
text = content['text']
|
||||||
|
# drop references:
|
||||||
|
# ^ The Penguin Dictionary
|
||||||
|
text = re.sub(r' \^ .*', '', text)
|
||||||
|
url = urlbase + 'wiki?curid=' + id
|
||||||
|
header = '<doc id="%s" url="%s" title="%s" language="%s" revision="%s">\n' % (id, url, title, language, revision)
|
||||||
|
page = header + title + '\n\n' + text + '\n</doc>\n'
|
||||||
|
output.write(page.encode('utf-8'))
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Minimum size of output files
|
||||||
|
minFileSize = 200 * 1024
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),
|
||||||
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||||
|
description=__doc__)
|
||||||
|
parser.add_argument("input",
|
||||||
|
help="Cirrus Json wiki dump file")
|
||||||
|
groupO = parser.add_argument_group('Output')
|
||||||
|
groupO.add_argument("-o", "--output", default="text",
|
||||||
|
help="directory for extracted files (or '-' for dumping to stdin)")
|
||||||
|
groupO.add_argument("-b", "--bytes", default="1M",
|
||||||
|
help="maximum bytes per output file (default %(default)s)",
|
||||||
|
metavar="n[KMG]")
|
||||||
|
groupO.add_argument("-c", "--compress", action="store_true",
|
||||||
|
help="compress output files using bzip")
|
||||||
|
|
||||||
|
groupP = parser.add_argument_group('Processing')
|
||||||
|
groupP.add_argument("-ns", "--namespaces", default="", metavar="ns1,ns2",
|
||||||
|
help="accepted namespaces")
|
||||||
|
|
||||||
|
groupS = parser.add_argument_group('Special')
|
||||||
|
groupS.add_argument("-q", "--quiet", action="store_true",
|
||||||
|
help="suppress reporting progress info")
|
||||||
|
groupS.add_argument("-v", "--version", action="version",
|
||||||
|
version='%(prog)s ' + version,
|
||||||
|
help="print program version")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
power = 'kmg'.find(args.bytes[-1].lower()) + 1
|
||||||
|
file_size = int(args.bytes[:-1]) * 1024 ** power
|
||||||
|
if file_size < minFileSize:
|
||||||
|
raise ValueError()
|
||||||
|
except ValueError:
|
||||||
|
logging.error('Insufficient or invalid size: %s', args.bytes)
|
||||||
|
return
|
||||||
|
|
||||||
|
FORMAT = '%(levelname)s: %(message)s'
|
||||||
|
logging.basicConfig(format=FORMAT)
|
||||||
|
|
||||||
|
logger = logging.getLogger()
|
||||||
|
if not args.quiet:
|
||||||
|
logger.setLevel(logging.INFO)
|
||||||
|
|
||||||
|
input_file = args.input
|
||||||
|
|
||||||
|
output_path = args.output
|
||||||
|
if output_path != '-' and not os.path.isdir(output_path):
|
||||||
|
try:
|
||||||
|
os.makedirs(output_path)
|
||||||
|
except:
|
||||||
|
logging.error('Could not create: %s', output_path)
|
||||||
|
return
|
||||||
|
|
||||||
|
process_dump(input_file, output_path, file_size, args.compress)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
29
wikiextractor/extract.sh
Normal file
29
wikiextractor/extract.sh
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# NOTES
|
||||||
|
#
|
||||||
|
# - Must expand templates to avoid a large loss of content.
|
||||||
|
# - Text will not (redundantly) contain the title string.
|
||||||
|
# - Keep sections. Section title will be marked by "Section::::".
|
||||||
|
# - Keep lists. List bullets will be marked by "BULLET::::".
|
||||||
|
# - Keep tables. They're mostly garbage but can be removed later (remove "^!*").
|
||||||
|
# - Remove disambiguation pages. Right now there is no use for them.
|
||||||
|
|
||||||
|
INPUT=$1
|
||||||
|
PROCESSES=$2
|
||||||
|
TEMPLATES=$3
|
||||||
|
OUTPUT=$4
|
||||||
|
|
||||||
|
python WikiExtractor.py $INPUT \
|
||||||
|
--json \
|
||||||
|
--processes $PROCESSES \
|
||||||
|
--templates $TEMPLATES \
|
||||||
|
--output $OUTPUT \
|
||||||
|
--bytes 1M \
|
||||||
|
--compress \
|
||||||
|
--links \
|
||||||
|
--sections \
|
||||||
|
--lists \
|
||||||
|
--keep_tables \
|
||||||
|
--min_text_length 0 \
|
||||||
|
--filter_disambig_pages
|
||||||
Loading…
Reference in New Issue
Block a user