forked from KEMT/zpwiki
		
	Upload files to 'pages/students/2016/jan_holp/dp2021'
najdôležitejší zdrojový súbor, slúži na indexovanie dát do ES
This commit is contained in:
		
							parent
							
								
									bb6a75ff45
								
							
						
					
					
						commit
						e5dcb538ea
					
				
							
								
								
									
										72
									
								
								pages/students/2016/jan_holp/dp2021/elasticsearch.js
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										72
									
								
								pages/students/2016/jan_holp/dp2021/elasticsearch.js
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,72 @@
 | 
			
		||||
 | 
			
		||||
const elasticsearch = require('elasticsearch');
 | 
			
		||||
const fs = require('fs');
 | 
			
		||||
const readline = require('readline');
 | 
			
		||||
 | 
			
		||||
const client = new elasticsearch.Client({
 | 
			
		||||
    hosts: [ 'http://localhost:9200']
 | 
			
		||||
});
 | 
			
		||||
 | 
			
		||||
client.indices.create({
 | 
			
		||||
    index: 'skweb'
 | 
			
		||||
}, function(error, response, status) {
 | 
			
		||||
    if (error) {
 | 
			
		||||
        console.log(error);
 | 
			
		||||
    } else {
 | 
			
		||||
        console.log("created a new index", response);
 | 
			
		||||
    }
 | 
			
		||||
});
 | 
			
		||||
 | 
			
		||||
const bulkIndex = function bulkIndex(index, type, data) {
 | 
			
		||||
	let bulkBody = [];
 | 
			
		||||
	id = 1;
 | 
			
		||||
const errorCount = 0;
 | 
			
		||||
	data.forEach(item => {
 | 
			
		||||
		bulkBody.push({
 | 
			
		||||
			index: {
 | 
			
		||||
				_index: index,
 | 
			
		||||
				_type:  type,
 | 
			
		||||
				_id :   id++,
 | 
			
		||||
			}
 | 
			
		||||
		});
 | 
			
		||||
		bulkBody.push(item);
 | 
			
		||||
	});
 | 
			
		||||
        console.log(bulkBody);
 | 
			
		||||
	client.bulk({body: bulkBody})
 | 
			
		||||
		.then(response => {
 | 
			
		||||
 | 
			
		||||
			response.items.forEach(item => {
 | 
			
		||||
				if (item.index && item.index.error) {
 | 
			
		||||
					console.log(++errorCount, item.index.error);
 | 
			
		||||
				}
 | 
			
		||||
			});
 | 
			
		||||
			console.log(
 | 
			
		||||
				`Successfully indexed ${data.length - errorCount}
 | 
			
		||||
				out of ${data.length} items`
 | 
			
		||||
			);
 | 
			
		||||
		})
 | 
			
		||||
		.catch(console.err);
 | 
			
		||||
};
 | 
			
		||||
 | 
			
		||||
async  function indexData() {
 | 
			
		||||
 | 
			
		||||
   let documents = [];
 | 
			
		||||
   const readInterface = readline.createInterface({
 | 
			
		||||
      input: fs.createReadStream('/home/elastic/BP/skweb/server/text.txt'),
 | 
			
		||||
//    output: process.stdout,
 | 
			
		||||
      console: false
 | 
			
		||||
   });
 | 
			
		||||
   readInterface.on('line', function(line) {  
 | 
			
		||||
      const article = JSON.parse(line);
 | 
			
		||||
      documents.push(article);
 | 
			
		||||
 | 
			
		||||
    });
 | 
			
		||||
      readInterface.on('close', function() {
 | 
			
		||||
        console.log(documents);  
 | 
			
		||||
	 bulkIndex('skweb', 'web_page', documents);
 | 
			
		||||
      });
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  indexData();
 | 
			
		||||
		Loading…
	
		Reference in New Issue
	
	Block a user