73 lines
1.5 KiB
JavaScript
73 lines
1.5 KiB
JavaScript
|
|
||
|
|
const elasticsearch = require('elasticsearch');
|
||
|
|
const fs = require('fs');
|
||
|
|
const readline = require('readline');
|
||
|
|
|
||
|
|
const client = new elasticsearch.Client({
|
||
|
|
hosts: [ 'http://localhost:9200']
|
||
|
|
});
|
||
|
|
|
||
|
|
client.indices.create({
|
||
|
|
index: 'skweb'
|
||
|
|
}, function(error, response, status) {
|
||
|
|
if (error) {
|
||
|
|
console.log(error);
|
||
|
|
} else {
|
||
|
|
console.log("created a new index", response);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
const bulkIndex = function bulkIndex(index, type, data) {
|
||
|
|
let bulkBody = [];
|
||
|
|
id = 1;
|
||
|
|
const errorCount = 0;
|
||
|
|
data.forEach(item => {
|
||
|
|
bulkBody.push({
|
||
|
|
index: {
|
||
|
|
_index: index,
|
||
|
|
_type: type,
|
||
|
|
_id : id++,
|
||
|
|
}
|
||
|
|
});
|
||
|
|
bulkBody.push(item);
|
||
|
|
});
|
||
|
|
console.log(bulkBody);
|
||
|
|
client.bulk({body: bulkBody})
|
||
|
|
.then(response => {
|
||
|
|
|
||
|
|
response.items.forEach(item => {
|
||
|
|
if (item.index && item.index.error) {
|
||
|
|
console.log(++errorCount, item.index.error);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
console.log(
|
||
|
|
`Successfully indexed ${data.length - errorCount}
|
||
|
|
out of ${data.length} items`
|
||
|
|
);
|
||
|
|
})
|
||
|
|
.catch(console.err);
|
||
|
|
};
|
||
|
|
|
||
|
|
async function indexData() {
|
||
|
|
|
||
|
|
let documents = [];
|
||
|
|
const readInterface = readline.createInterface({
|
||
|
|
input: fs.createReadStream('/home/elastic/BP/skweb/server/text.txt'),
|
||
|
|
// output: process.stdout,
|
||
|
|
console: false
|
||
|
|
});
|
||
|
|
readInterface.on('line', function(line) {
|
||
|
|
const article = JSON.parse(line);
|
||
|
|
documents.push(article);
|
||
|
|
|
||
|
|
});
|
||
|
|
readInterface.on('close', function() {
|
||
|
|
console.log(documents);
|
||
|
|
bulkIndex('skweb', 'web_page', documents);
|
||
|
|
});
|
||
|
|
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
indexData();
|