forked from KEMT/zpwiki
Upload files to 'pages/students/2016/jan_holp/dp2021'
najdôležitejší zdrojový súbor, slúži na indexovanie dát do ES
This commit is contained in:
parent
bb6a75ff45
commit
e5dcb538ea
72
pages/students/2016/jan_holp/dp2021/elasticsearch.js
Normal file
72
pages/students/2016/jan_holp/dp2021/elasticsearch.js
Normal file
@ -0,0 +1,72 @@
|
||||
|
||||
const elasticsearch = require('elasticsearch');
|
||||
const fs = require('fs');
|
||||
const readline = require('readline');
|
||||
|
||||
const client = new elasticsearch.Client({
|
||||
hosts: [ 'http://localhost:9200']
|
||||
});
|
||||
|
||||
client.indices.create({
|
||||
index: 'skweb'
|
||||
}, function(error, response, status) {
|
||||
if (error) {
|
||||
console.log(error);
|
||||
} else {
|
||||
console.log("created a new index", response);
|
||||
}
|
||||
});
|
||||
|
||||
const bulkIndex = function bulkIndex(index, type, data) {
|
||||
let bulkBody = [];
|
||||
id = 1;
|
||||
const errorCount = 0;
|
||||
data.forEach(item => {
|
||||
bulkBody.push({
|
||||
index: {
|
||||
_index: index,
|
||||
_type: type,
|
||||
_id : id++,
|
||||
}
|
||||
});
|
||||
bulkBody.push(item);
|
||||
});
|
||||
console.log(bulkBody);
|
||||
client.bulk({body: bulkBody})
|
||||
.then(response => {
|
||||
|
||||
response.items.forEach(item => {
|
||||
if (item.index && item.index.error) {
|
||||
console.log(++errorCount, item.index.error);
|
||||
}
|
||||
});
|
||||
console.log(
|
||||
`Successfully indexed ${data.length - errorCount}
|
||||
out of ${data.length} items`
|
||||
);
|
||||
})
|
||||
.catch(console.err);
|
||||
};
|
||||
|
||||
async function indexData() {
|
||||
|
||||
let documents = [];
|
||||
const readInterface = readline.createInterface({
|
||||
input: fs.createReadStream('/home/elastic/BP/skweb/server/text.txt'),
|
||||
// output: process.stdout,
|
||||
console: false
|
||||
});
|
||||
readInterface.on('line', function(line) {
|
||||
const article = JSON.parse(line);
|
||||
documents.push(article);
|
||||
|
||||
});
|
||||
readInterface.on('close', function() {
|
||||
console.log(documents);
|
||||
bulkIndex('skweb', 'web_page', documents);
|
||||
});
|
||||
|
||||
|
||||
}
|
||||
|
||||
indexData();
|
Loading…
Reference in New Issue
Block a user