Ego_Search_Elastic.php

gehe zur Dokumentation dieser Datei

<?php 
require_once('base/Ego_Search.php');
require_once('composer/vendor/autoload.php');
 
class Ego_Search_Elastic extends Ego_Search
{
    protected $client = null;
 
    private $officeImport = false;
 
    protected $maxClauseCount = 10000;
 
    private $throwException = false;
 
    public function optimize() {
        // no function
    }
 
    public function __construct($table = '', $param = [], bool $checkHealthy = false) {
        $this->throwException = $GLOBALS['egotec_conf']['elastic']['throw_exception'] || $param['throw_exception'];
 
        if (!Ego_System::checkLicence($GLOBALS['egotec_conf']['lib_dir'] . 'elastic')) {
            throw new Exception("missing licence");
        }
 
        $this->officeImport = Ego_System::checkLicence($GLOBALS['egotec_conf']['lib_dir'] . 'office');
 
        if ($GLOBALS['egotec_conf']['elastic']['maxclause']) {
            $this->maxClauseCount = (int) $GLOBALS['egotec_conf']['elastic']['maxclause'];
        }
 
        $this->config = [
            'index' => '', // Pfad für den Index (demo_de / wiki_de / multimedia_de)
            'leading_wildcard' => true
        ];
 
        if (empty($table)) {
            $table = $GLOBALS['site']->pageTable;
        }
 
        if (!$GLOBALS['egotec_conf']['elastic']['max_results']) {
            $GLOBALS['egotec_conf']['elastic']['max_results'] = 10000;
        }
 
        // examples host
        // https://username:password@foo.com:9200/
        // http://foo.com:9200/
 
        $hosts = [];
        for ($i = 0; $i < 4; $i++) {
            if ($GLOBALS['egotec_conf']['elastic']['host' . $i]) {
                $hosts[] = trim($GLOBALS['egotec_conf']['elastic']['host' . $i], '/');
            }
        }
 
        if (empty($hosts)) {
            throw new Exception("missing hosts for elastic");
        }
 
        $this->client = Elasticsearch\ClientBuilder::create()
            ->setHandler(Elasticsearch\ClientBuilder::multiHandler())
            ->setHosts($hosts)
            ->build();
 
        $this->config['index'] = strtolower($table);
        $this->config['table'] = $table;
        $this->config['param'] = $param;
 
 
        try {
            $this->indexCreate();
        } catch (Exception $e) {
            if ($checkHealthy) {
                throw $e;
            } else {
                $this->error($e);
            }
        }
 
        $this->createPipeline();
    }
 
    public function delete($id) {
        $params = [
            'index' => $this->config['index'],
            'id' => $id
        ];
 
        $this->client->delete($params);
 
        return true;
    }
 
    public function reset() {
        $GLOBALS['monitor']['search_reset']++;
 
        $this->indexDelete(false);
 
        try {
            $this->indexCreate();
        } catch (Exception $e) {
            $this->error($e);
        }
 
        return true;
    }
 
 
    public function resetAll() {
        $GLOBALS['monitor']['search_reset_all']++;
 
        $this->indexDelete(true);
 
        return true;
    }
 
    public function indexCreate() {
        $params = [
            'index' => $this->config['index']
        ];
 
        if (!$this->client->indices()->exists($params)) {
            $language = preg_match('/_([^_]{2})$/', $params['index'], $matches) ? $matches[1] : '';
            $stopwords = [
                'ar' => '_arabic_',
                'br' => '_brazilian_',
                'bg' => '_bulgarian_',
                'cn' => '_cjk_',
                'zh' => '_cjk_',
                'ja' => '_cjk_',
                'ko' => '_cjk_',
                'cs' => '_czech_',
                'da' => '_danish_',
                'nl' => '_dutch_',
                'en' => '_english_',
                'us' => '_english_',
                'uk' => '_english_',
                'fi' => '_finnish_',
                'fr' => '_french_',
                'de' => '_german_',
                'ed' => '_german_',
                'el' => '_greek_',
                'hi' => '_hindi_',
                'hu' => '_hungarian_',
                'id' => '_indonesian_',
                'ga' => '_irish_',
                'it' => '_italian_',
                'lv' => '_latvian_',
                'lt' => '_lithuanian_',
                'no' => '_norwegian_',
                'pt' => '_portuguese_',
                'ro' => '_romanian_',
                'ru' => '_russian_',
                'es' => '_spanish_',
                'sv' => '_swedish_',
                'th' => '_thai_',
                'tr' => '_turkish_'
            ];
 
            $sortable_fields = function() {
                $arr = [];
                foreach ([
                    'id' => 'long',
                    'name' => 'text',
                    'type' => 'text',
                    'a_date' => 'date',
                    'c_date' => 'date',
                    'release_from' => 'date',
                    'release_until' => 'date',
                    'order_field' => 'long',
                    'structure' => 'nested',
                    'main_category' => 'long'
                ] as $name => $type) {
                    $params = ['type' => $type];
                    if ($name == 'structure') {
                        $params['properties'] = [
                            'parent' => [
                                'type' => 'integer',
                                'fields' => [
                                    'keyword' => [
                                        'type' => 'keyword'
                                    ]
                                ]
                            ],
                            'position' => [
                                'type' => 'integer',
                                'fields' => [
                                    'keyword' => [
                                        'type' => 'keyword'
                                    ]
                                ]
                            ]
                        ];
                    } else {
                        $params['fields'] = [
                            'keyword' => [
                                'type' => 'keyword'
                            ]
                        ];
                    }
                    if ($type == 'date') {
                        $params['format'] = 'epoch_second';
                    }
                    $arr[$name] = $params;
                }
                return $arr;
            };
            $params = [
                'index' => $this->config['index'],
                'include_type_name' => true,
                'body' => [
                    'settings' => [
                        'number_of_shards' => $GLOBALS['egotec_conf']['elastic']['number_of_shards'],
                        'number_of_replicas' => $GLOBALS['egotec_conf']['elastic']['number_of_replicas'] ?: 0,
                        'index.mapping.ignore_malformed' => true,
                        'index.mapping.total_fields.limit' => $this->maxClauseCount,
                        'analysis' => [
                            'analyzer' => [
                                'default' => [
                                    'type' => 'custom',
                                    'tokenizer' => 'standard',
                                    'filter' => ['lowercase', 'stop']
                                ],
                                'trigram' => [
                                    'type' => 'custom',
                                    'tokenizer' => 'standard',
                                    'filter' => ['lowercase', 'shingle']
                                ]
                            ],
                            'filter' => [
                                'shingle' => [
                                    'type' => 'shingle',
                                    'min_shingle_size' => 2,
                                    'max_shingle_size' => 3
                                ],
                                'stop' => [
                                    'type' => 'stop',
                                    'ignore_case' => true,
                                    'stopwords' => $stopwords[$language] ?? $stopwords['en']
                                ]
                            ]
                        ]
                    ],
                    'mappings' => [
                        '_doc' => [
                            'properties' => array_merge($sortable_fields(), [
                                'suggest' => [
                                    'type' => 'completion',
                                    'contexts' => [[
                                        'name' => 'ignore_search',
                                        'type' => 'category',
                                        'path' => 'ignore_search'
                                    ]]
                                ],
                                'title' => [
                                    'type' => 'text',
                                    'fields' => [
                                        'keyword' => [
                                            'type' => 'keyword'
                                        ],
                                        'trigram' => [
                                            'type' => 'text',
                                            'analyzer' => 'trigram'
                                        ]
                                    ]
                                ]
                            ])
                        ]
                    ]
                ]
            ];
 
            try {
                $this->client->indices()->create($params);
            } catch (Exception $e) {
                if (strpos($e->getMessage(), 'resource_already_exists_exception') === false) {
                    throw new Exception($e->getMessage(), $e->getCode());
                }
            }
        }
 
        return true;
    }
 
    public function indexExists(): bool {
        return $this->client->indices()->exists([
            'index' => $this->config['index']
        ]);
    }
 
    public function indexDelete($all) {
        $params = [
            'index' => $all ? "_all" : $this->config['index'],
            'client' => ['ignore' => [400, 404]]
        ];
 
        if ($this->client->indices()->exists($params)) {
            $this->client->indices()->delete($params);
        }
 
        return true;
    }
 
    public function updateBulk($pages, $recursive = true) {
        try {
            foreach ($pages as $page) {
                $page = $this->indexFiles($page);
 
                $params['body'][] = [
                    'index' => [
                        '_index' => $this->config['index'],
                        '_id' => $page->field['id']
                    ]
                ];
 
                $params['body'][] = $this->getBody($page);
 
                // Für alle untergeordneten Seiten die Strukturen aktualisieren (falls sich die variable Sortierung geändert hat)
                if ($recursive) {
                    foreach ($page->getChildren([], ['auth_or' => '1=1']) as $child) {
                        $child->updateIndex($this, true, false);
                    }
                }
            }
 
            $this->client->bulk($params);
        } catch (Exception $e) {
            foreach ($pages as $page) {
                Ego_Action::add(Ego_Action::PAGE_UPDATE_INDEX, [
                    'identity' => $page->getIdentity(),
                    'recursive' => $recursive
                ], $e);
            }
            Ego_Action::throw($e);
        }
    }
 
    public function update($index, $page, $count = [], $recursive = true) {
        try {
            $page = $this->indexFiles($page);
 
            $GLOBALS['monitor']['search_update_count']++;
 
            $params = [
                'index' => $this->config['index'],
                'id' => $index,
                'body' => $this->getBody($page),
                'client' => ['ignore' => 404]
            ];
 
            $this->client->index($params);
        } catch (Exception $e) {
            Ego_Action::add(Ego_Action::PAGE_UPDATE_INDEX, [
                'identity' => $page->getIdentity(),
                'recursive' => $recursive
            ], $e, true);
        }
 
        // Für alle untergeordneten Seiten die Strukturen aktualisieren (falls sich die variable Sortierung geändert hat)
        if ($recursive) {
            foreach ($page->getChildren([], ['auth_or' => '1=1']) as $child) {
                $child->updateIndex($this, true, false);
            }
        }
 
        return true;
    }
 
    private function getBody($page) {
        $page = clone $page;
 
        $keywords = $this->_getContent($page, 'keywords');
        $name = $this->_getContent($page, 'name');
        $title = $this->_getContent($page, 'title');
        $short = $this->_getContent($page, 'short');
        $content = $this->_getContent($page, 'content');
 
        // Blöcke aus dem Extrafeld nicht doppelt in die Suche übernehmen. Diese sind bereits in "content".
        $extra = $this->_getExtra($page);
        unset($page->extra['_contents']);
        $extra_values = $this->_getExtra($page, true);
 
        $suggestions = array_values(
            array_unique(
                array_filter(
                    array_map(
                        function ($value) {
                            return Ego_System::filterNonUtf8(trim(
                                preg_replace('/[^a-zA-Z0-9äöüÄÖÜß-]/', '', $value),
                                '-'
                            ));
                        },
                        explode(
                            ' ',
                            mb_strtolower(
                                html_entity_decode(
                                    implode(
                                        ' ',
                                        [$keywords, $name, $title, $short, $content]
                                    )
                                )
                            )
                        )
                    ),
                    function ($value) {
                        return
                            !is_numeric($value)
                            && strlen($value) > 2
                            && !preg_match('/^[a-z0-9]{32}$/i', $value) // Keine Hashes
                        ;
                    }
                )
            )
        );
 
        $convert_date = function($date) {
            if (!$date || $date == '0000-00-00 00:00:00') {
                return null;
            }
            return strtotime($date);
        };
 
        // Die Struktur ermitteln, damit die variable Sortierung abgefragt werden kann
        $structure = [];
        $db = new_db_connection([
            'table' => $page->getSite()->pageTable . '_children',
            'where' => 'child = :id',
            'bind' => [
                'id' => $page->field['id']
            ]
        ]);
        while ($db->nextRecord()) {
            $structure[] = [
                'parent' => $db->Record['page_id'],
                'position' => $db->Record['idx']
            ];
        }
 
        return [
            'id' => $page->field['id'],
            'keywords' => $keywords,
            'url' => $this->_getContent($page, 'url'),
            'name' => $name,
            'title' => $title,
            'short' => $short,
            'content' => $content,
            'a_date' => $convert_date($page->field['a_date']),
            'c_date' => $convert_date($page->field['c_date']),
            'release_from' => $convert_date($page->field['release_from']),
            'release_until' => $convert_date($page->field['release_until']),
            'extra' => $extra,
            'extra_values' => trim(
                implode(
                    ' ',
                    array_values(
                        array_map(
                            function ($value) {
                                return implode(' ', $value);
                            },
                            $extra_values
                        )
                    )
                )
            ),
            'type' => $page->field['type'],
            'ignore_search' => (($page->field['nav_hide'] & 4) == 4) ? '1' : '0',
            'inactive' => (int)$page->field['inactive'],
            'deleted' => (int)$page->field['deleted'],
            'order_field' => (int)$page->field['order_field'],
            'structure' => $structure,
            'main_category' => (int)$page->extra['main_category'],
            'suggest' => [
                'input' => $page->isActive() ? $suggestions : []
            ]];
    }
 
    private function getSearchParam($tables, $search, $filter, $fuzzy, $id_list = [], $limit = '', $order = '', $where = '') {
        $fields = ['keywords', 'url', 'name', 'title', 'short', 'content', 'extra_values'];
        $indexes = $tables ? $tables : [$this->config['index']];
 
        // Verwendete "rewrite" Einstellung
        $rewrite = 'top_terms_' . $this->maxClauseCount;
 
        // Immer alles klein verwenden: Elastic startet eine Case Sensitive Suche, wenn ein Großbuchstabe im Suchbegriff vorkommt
        $search = trim(mb_strtolower($search));
 
        // Suchbegriff automatisch korrigieren
        $search = trim(preg_replace([
            '/[?*+-]( |$)/',
            '/(^| )\{(.*?)\}/'
        ], [
            ' ',
            '\\1\\2'
        ], $search));
 
        // Stop Wörter entfernen
        if (empty($GLOBALS['elastic']['stop_words'])) {
            $search_phrases = [];
            $search_no_phrases = $search; // Suchbegriff ohne Phrasen
            if (preg_match_all('/".*?"/', $search, $matches)) {
                $search_phrases = $matches[0];
                $search_no_phrases = trim(preg_replace(['/".*?"/', '/\s+/'], ['', ' '], $search)); // Suchbegriff ohne Phrasen
            }
 
            // Stop Wörter ermitteln
            $language = $GLOBALS['site'] ? $GLOBALS['site']->language : 'de';
            if (Ego_System::file_exists($file = $GLOBALS['egotec_conf']['lib_dir'] . 'base/search/' . $language . '.txt')) {
                $stop_words = array_filter(explode("\n", Ego_System::file_get_contents($file)));
 
                $search_no_phrases = implode(' ', array_filter(explode(' ', $search_no_phrases), function($value) use ($stop_words) {
                    return !in_array($value, $stop_words);
                }));
                $search = $search_no_phrases . (!empty($search_phrases) ? ' ' . implode(' ', $search_phrases) : '');
            }
        }
 
        /* Suchbegriffe mit Bindestrich automatisch um die einzelnen Begriffe erweitern (diese werden sonst nicht gefunden)
         * Ausnahme: die Bindestriche kommen in einer Phrase vor (innerhalb von Hochkommata) */
        $search_no_phrases = trim(preg_replace(['/".*?"/', '/\s+/'], ['', ' '], $search)); // Suchbegriff ohne Phrasen
        if (preg_match_all('/( |^)([^+-][^ "]+-[^ "]+)/is', $search_no_phrases, $matches)) {
            foreach ($matches[2] as $word) {
                $search .= ' ' . str_replace('-', ' ', $word);
            }
        }
 
        // Suchrelevanz für Ein-Mandanten-Suche hinzufügen
        $count = [];
        if ($tables === null) {
            try {
                [$name, $lang] = preg_split('/_(?=[^_]*$)/', $this->config['table']);
                $site = new Site($name, $lang);
                $count = $site->getSearchCount();
            } catch (Site_Exception $e) {
                // ignorieren
            }
        }
 
        $must = [];
        $must_not = [];
        $should = [];
        $must_should = [];
 
        if (preg_match_all('/(".*?"|[^ ]+)/is', $search, $matches)) {
            // Zusätzliche Phrasen-Suche ausführen
            $match_phrase = sizeof($matches[0]) > 1 && !preg_match('/(^| )["*+-]/', $search);
 
            foreach ($fields as $field) {
                $boost = 1;
                if ($field == 'extra_values' && isset($count['extra'])) {
                    $boost = (int)$count['extra'];
                } elseif (isset($count[$field])) {
                    $boost = (int)$count[$field];
                }
 
                // Zu kurze Suchbegriffe entfernen (Standard Mindestlänge: 3 Zeichen)
                $filtered_query = array_filter($matches[0], function($s) {
                    return mb_strlen(trim($s, '"+-*')) >= (isset($GLOBALS['egotec_conf']['elastic']['minlength'])
                            ? (int) $GLOBALS['egotec_conf']['elastic']['minlength']
                            : 3);
                });
 
                foreach ($filtered_query as $query) {
                    // Hotfix: Enthält der Suchbegriff ein "-", muss dieser eine exakte Suche auslösen
                    if (!in_array($query[0], ['"', '-']) && strpos($query, '-') !== false) {
                        $query = "\"$query\"";
                    }
 
                    if ($query[0] == '"') {
                        // Phrase
                        $query = trim($query, '"');
                        $should[] = [
                            'match_phrase' => [
                                $field => [
                                    'query' => $query,
                                    'boost' => $boost
                                ]
                            ]
                        ];
                    } else {
                        if ($fuzzy) {
                            // Fuzzy
                            $should[] = [
                                'fuzzy' => [
                                    $field => [
                                        'value' => $query,
                                        'rewrite' => $rewrite,
                                        'boost' => $boost
                                    ]
                                ]
                            ];
                        } elseif (preg_match('/^(.*?)~([0-9.]+|)$/', $query, $match)) {
                            // Fuzzy für diesen Suchbegriff
                            $query = $match[1];
                            $fuzziness = floatval($match[2] !== '' ? $match[2] : 1);
 
                            $should[] = [
                                'fuzzy' => [
                                    $field => [
                                        'value' => $query,
                                        'rewrite' => $rewrite,
                                        'boost' => $boost,
                                        'fuzziness' => round(2 * $fuzziness)
                                    ]
                                ]
                            ];
                        } else {
                            // Wildcard
                            $wildcard = function ($query) {
                                if (strpos($query, '*') === false) {
                                    $query = "*$query*";
                                }
                                return $query;
                            };
 
                            switch ($query[0]) {
                                case '+':
                                    $query = ltrim($query, '+');
                                    $must_should[md5($query)][] = [
                                        'wildcard' => [
                                            $field => [
                                                'value' => $wildcard($query),
                                                'rewrite' => $rewrite,
                                                'boost' => $boost
                                            ]
                                        ]
                                    ];
                                    break;
                                case '-':
                                    $query = ltrim($query, '-');
                                    $must_not[] = [
                                        'wildcard' => [
                                            $field => [
                                                'value' => $wildcard($query),
                                                'rewrite' => $rewrite,
                                                'boost' => $boost
                                            ]
                                        ]
                                    ];
                                    break;
                                default:
                                    $should[] = [
                                        'wildcard' => [
                                            $field => [
                                                'value' => $wildcard($query),
                                                'rewrite' => $rewrite,
                                                'boost' => $boost
                                            ]
                                        ]
                                    ];
                            }
                        }
                    }
                }
 
                // Ein Suchbegriff mit mehreren Wörtern erzeugt gleichzeitig auch eine bevorzugte Phrasen-Suche
                if ($match_phrase) {
                    $should[] = [
                        'match_phrase' => [
                            $field => [
                                'query' => trim($search),
                                'boost' => $boost + 1
                            ]
                        ]
                    ];
                }
            }
        }
 
        $minimum_should_match = $search == '' && (!empty($filter) || !empty($this->extraQuery)) ? 0 : 1;
 
        if (sizeof($must_should)) {
            $minimum_should_match = 0;
 
            foreach ($must_should as $items) {
                $must[] = [
                    'bool' => [
                        'should' => $items,
                        'minimum_should_match' => 1
                    ]
                ];
            }
        }
 
        if ($this->config['param']['only_active']) {
            $must[] = [
                'match' => [
                    'inactive' => 0
                ]
            ];
        }
        if (!$this->config['param']['deleted'] && !$this->config['param']['deleted_or']) {
            $must[] = [
                'match' => [
                    'deleted' => 0
                ]
            ];
        }
        if ($this->config['param']['search']) {
            $must[] = [
                'match' => [
                    'ignore_search' => '0'
                ]
            ];
        }
 
        // Nur Treffer in einer ID Liste ermitteln
        if (!empty($id_list)) {
            $must[] = [
                'constant_score' => [
                    'filter' => [
                        'terms' => [
                            '_id' => $id_list
                        ]
                    ]
                ]
            ];
        }
 
        // Extra-Suche hinzufügen
        if (!empty($this->extraQuery)) {
            $must[] = [
                'query_string' => [
                    'query' => $this->extraQuery
                ]
            ];
        }
 
        // Filter ermitteln
        if (!empty($filter)) {
            if ($search == '') {
                /* Ohne Suchbegriff liefert die Suche keine Ergebnisse.
                 * Der Filter wird dann zum Suchkriterium. */
                $must[] = [
                    'query_string' => [
                        'query' => $filter
                    ]
                ];
                $filter = [];
            } else {
                $filter = [
                    'query_string' => [
                        'query' => $filter
                    ]
                ];
            }
        } else {
            $filter = [];
        }
 
        // Limit ermitteln (standardmäßig werden alle Ergebnisse zurückgegeben)
        if ($limit) {
            $limit_parts = explode(',', $limit);
 
            if (count($limit_parts) > 1) {
                $from = trim($limit_parts[0]);
                $size = trim($limit_parts[1]);
            } else {
                $from = 0;
                $size = trim($limit_parts[0]);
            }
        } else {
            $from = 0;
            $size = (int) $GLOBALS['egotec_conf']['elastic']['max_results'];
        }
 
        // Sortierung ermitteln
        $sort = [];
        $score_sorted = false;
 
        if ($order) {
            foreach (explode(',', $order) as $order_part) {
                $parts = explode(' ', trim($order_part), 2);
                $key = trim($parts[0]);
 
                // Wenn nach der Relevanz der Ergebnisse sortiert wird, darf nicht nach "_score.keywords" sortiert werden
                if ($key === 'score') {
                    $sort[] = ['_score' => ['order' => $parts[1]]];
                    $score_sorted = true;
                    continue;
                }
 
                $parent_id = null;
                if (strpos($key, '.') !== false) {
                    foreach ($indexes as $index) {
                        if (strpos($key, "{$index}.") === 0) {
                            $key = substr($key, strlen("{$index}."));
                            break;
                        } elseif (
                            $where
                            && $key == "{$index}_children.idx"
                            && preg_match('/' . $index . '_children\.page_id\s*=\s*(\d+)/', $where, $match)
                        ) {
                            $key = 'structure.position';
                            $parent_id = $match[1];
                            break;
                        } else {
                            continue 2;
                        }
                    }
                }
                if (!preg_match('/\.keyword$/i', $key)) {
                    $key .= '.keyword';
                }
 
                // Weitere Parameter für bestimmte Felder
                $params = [];
                if ($parent_id && strpos($key, 'structure.') === 0) {
                    $params['nested'] = [
                        'path' => 'structure',
                        'filter' => [
                            'term' => ['structure.parent' => $parent_id]
                        ]
                    ];
                }
 
                $sort[] = [
                    $key => array_merge([
                        'order' => strtolower(trim($parts[1]))
                    ], $params)
                ];
            }
        }
 
        // Wenn nicht bereits nach "_score" sortiert wird
        if (!$score_sorted) {
            $sort[] = ['_score' => [
                'order' => 'desc'
            ]];
        }
 
        // Standardmäßig immer nach der ID sortieren (relevant, wenn der "_score" identisch ist)
        $sort[] = ['id' => [
            'order' => 'asc'
        ]];
 
        return [
            'index' => $indexes,
            // KEINE Inhalte zurückliefern
            '_source' => false,
            'track_scores' => true,
            'body' => [
                'from' => $from,
                'size' => $size,
                'sort' => $sort,
                'query' => [
                    'bool' => [
                        'must' => $must,
                        'must_not' => $must_not,
                        'should' => $should,
                        'filter' => $filter,
                        'minimum_should_match' => $minimum_should_match
                    ]
                ]
            ]
        ];
    }
 
    private function createPipeline() {
        if ($this->officeImport) {
            $params = [
                'id' => 'attachment',
                'body' => [
                    'description' => 'Extract attachment information',
                    'processors' => [
                        [
                            'attachment' => [
                                'field' => 'base64',
                                'indexed_chars' => -1
                            ]
                        ]
                    ]
                ]
            ];
 
            $this->client->ingest()->putPipeline($params);
        }
    }
 
    protected function indexFile(Page $page, $path) {
        if (!$this->officeImport || $page->extra['quarantine'] || !Ego_System::file_exists($path)) {
            return null;
        }
 
        @ini_set("memory_limit", "-1");
 
        $pageTable = strtolower($page->getSite()->pageTable);
        $identity = $page->getIdentity();
 
        $params = [
            'index' => $pageTable,
            'id' => $identity,
            'pipeline' => 'attachment',
            'body' => [
                'base64' => base64_encode(Ego_System::file_get_contents($path, false))
            ]
        ];
 
        try {
            $result = $this->client->index($params);
        } catch (Exception $e) {
            egotec_error_log('Elasticsearch: error while indexing file of ' . $page->getIdentity() . ' (' . $e->getMessage() . ')');
            return null;
        }
 
        if ($result) {
            $content = $this->getIndexFile($pageTable, $identity);
            $this->delete($identity);
 
            return Ego_System::stringEncode($content);
        }
 
        return null;
    }
 
    private function getIndexFile($pageTable, $identity) {
        $params = [
            'index' => strtolower($pageTable),
            'id' => $identity
        ];
 
        $response = $this->client->get($params);
        return $response['_source']['attachment']['content'];
    }
 
    private function indexFiles(Page $page) {
        if (!$this->officeImport || $page->extra['quarantine']) {
            return $page;
        }
 
        if (
            $page->field['type'] == 'multimedia/file'
            && !$page->extra['_indexed']
        ) {
            if (!preg_match('/(image|video|audio|zip|exe|rar|octet-stream|postscript)/is', $page->extra['mime_type'])) {
                $page->extra['_indexed'] = true;
                try {
                    $content = $this->indexFile($page, $GLOBALS['egotec_conf']['var_dir'] . 'media/' . $page->getSite()->name . '/' . $page->getMediaFilename());
                    $page->field['content'] = Ego_System::filterNonUtf8($content, '', true);
                } catch (Exception $e) {
                    // Datei kann nicht indiziert werden
                    if (
                        strpos($e->getMessage(), 'EncryptedDocumentException') === false
                        && strpos($e->getMessage(), 'TikaException') === false
                    ) {
                        egotec_error_log('Elasticsearch: error while indexing file of ' . $page->getIdentity() . ' (' . $e->getMessage() . ')');
                    }
                }
 
                $page->update([], true, true);
            }
        }
 
        return $page;
    }
 
    public function search($search, $relation, $query, $filter = '', $fuzzy = false) {
        $this->checkSearch($search);
 
        $GLOBALS['monitor']['search_count']++;
        $GLOBALS['monitor']['search_length'] += mb_strlen($search);
        $GLOBALS['monitor']['search_words'] += substr_count($search, ' ') + 1;
 
        $params = $this->getSearchParam(null, $search, $filter, $fuzzy, $query['id_list'], $query['limit'] ?? '', $query['order'] ?? '', $query['where'] ?? '');
 
        $start = microtime(true);
        try {
            $results = $this->client->search($params);
        } catch (Exception $e) {
            $this->error($e);
        }
 
        $hits = $results["hits"]["hits"];
        $stop = microtime(true);
 
        $duration = (int)(($stop - $start) * 1000);
        $GLOBALS['monitor']['search_duration'] += $duration;
 
        if (empty($hits)) {
            unset($query['order']);
            unset($query['limit']);
            unset($query['bind']);
            $query['where'] = '1=0';
            return $query;
        }
 
        $ids = [];
 
        foreach ($hits as $key => $value) {
            $ids[$value["_id"]] = $query['order'] ? count($hits) - $key : $value["_score"];
        }
 
        if ($query['order']) {
            $min_value = 0;
            $multiply = 1;
        } else {
            $max_value = $results["hits"]["max_score"];
            $min_value = min(array_values($ids));
            if ($max_value == $min_value) {
                $multiply = 4 / $max_value;
                $min_value = 0;
            } else {
                $multiply = 4 / ($max_value - $min_value);
            }
        }
 
        return $this->buildQuery($ids, $relation, $query, $min_value, $multiply);
    }
 
    public function globalSearch($search, $sites = [], $query = [], $param = [], $sort = [], $filter = '') {
        $this->checkSearch($search);
 
        $start1 = microtime(true);
        $GLOBALS['monitor']['search_global_count']++;
        $GLOBALS['monitor']['search_global_length'] += mb_strlen($search);
        $GLOBALS['monitor']['search_global_words'] += substr_count($search, ' ') + 1;
 
        // Standardmäßig im Frontend nur Seiten anzeigen, die auch von der Suche gefunden werden dürfen
        if (!isset($param['search']) && empty($GLOBALS['admin_area'])) {
            $param['search'] = true;
        }
 
        // Betroffene Tabellen ermitteln
        $lang = $_REQUEST['lang'] ? $_REQUEST['lang'] : ($GLOBALS['site'] ? $GLOBALS['site']->language : null);
        $relations = [];
        if (empty($sites)) {
            // keine bestimmten Mandanten, also alle durchsuchen
            $sites = Ego_System::getAllSites();
        }
        $tables = $this->getTables($sites, $lang, $relations);
 
        // Suche durchführen
        $start2 = microtime(true);
        $params = $this->getSearchParam($tables, $search, $filter, (bool)$param['fuzzy']);
        try {
            $results = $this->client->search($params);
        } catch (Exception $e) {
            $this->error($e);
        }
        $stop2 = microtime(true);
        $duration = (int)(($stop2 - $start2) * 1000);
        $GLOBALS['monitor']['search_global_d2'] += $duration;
 
        $hits = $results["hits"]["hits"];
 
        if (empty($hits)) {
            return [];
        }
 
        $min_value = PHP_INT_MAX;
 
        $sorted_hits = [];
        foreach ($hits as $key => $hit) {
            if (!isset($sorted_hits[$hit['_index']])) {
                $sorted_hits[$hit['_index']] = [];
            }
            $sorted_hits[$hit['_index']][$hit['_id']] = $query['order'] ? count($hits) - $key : $hit['_score'];
            if ($hit['_score'] < $min_value) {
                $min_value = $hit['_score'];
            }
        }
 
        if ($query['order']) {
            $min_value = 0;
            $multiply = 1;
        } else {
            $max_value = $results["hits"]["max_score"];
            if ($max_value == $min_value) {
                $multiply = 4 / $max_value;
                $min_value = 0;
            } else {
                $multiply = 4 / ($max_value - $min_value);
            }
        }
 
        // Treffer sammeln und sortiert zurückliefern
        $pages = [];
        foreach ($sorted_hits as $table => $ids) {
            $site = $relations[$table];
            foreach ($site->getPages($this->buildQuery($ids, $site->pageTable . '.id', $query, $min_value, $multiply), $param) as $page) {
                $pages[] = $page;
            }
        }
        $pages = $this->sortPages($pages, $query['order'], $sort);
 
        $stop1 = microtime(true);
        $duration = (int)(($stop1 - $start1) * 1000);
        $GLOBALS['monitor']['search_global_d1'] += $duration;
 
        return $pages;
    }
 
    public function getSuggestions($query, $sites = [], $max = 5) {
        $tables = $this->config['index'];
        if (!empty($sites)) {
            $tables = $this->getTables($sites);
        }
        try {
            $results = $this->client->search([
                'index' => $tables,
                // KEINE Inhalte zurückliefern
                '_source' => false,
                'sort' => ['_score'], // The order defaults to desc when sorting on the _score, and defaults to asc when sorting on anything else.
                'body' => [
                    'suggest' => [
                        'suggest' => [
                            'prefix' => $query,
                            'completion' => [
                                'field' => 'suggest',
                                'skip_duplicates' => true,
                                'size' => $max,
                                'contexts' => [
                                    'ignore_search' => ['0']
                                ]
                            ]
                        ]
                    ]
                ]
            ]);
        } catch (Exception $e) {
            $this->error($e);
        }
 
        $suggestions = [];
        if (is_array($results['suggest']['suggest'][0]['options'])) {
            foreach ($results['suggest']['suggest'][0]['options'] as $result) {
                if (mb_strtolower($query) != mb_strtolower($result['text'])) {
                    if (preg_match('/^(.*?)_([^_]+)$/', $result['_index'], $match)) {
                        // Suchvorschläge nur von Seiten liefern, die man sehen darf
                        try {
                            $site = new Site($match[1], $match[2]);
                            $page = $site->getPage($result['_id']);
 
                            if ($page && $page->hasRights(['view'])) {
                                $suggestions[] = $result['text'];
                            }
                        } catch (Site_Exception $e) {
                            // ignorieren
                        }
                    } else {
                        $suggestions[] = $result['text'];
                    }
                }
            }
        }
 
        return $suggestions;
    }
 
    public function getCorrections($query, $sites = [], $max = 3) {
        $tables = $this->config['index'];
        if (!empty($sites)) {
            $tables = $this->getTables($sites);
        }
        try {
            $results = $this->client->search([
                'index' => $tables,
                'body' => [
                    'suggest' => [
                        'text' => $query,
                        'simple_phrase' => [
                            'phrase' => [
                                'field' => 'title.trigram',
                                'size' => $max,
                                'gram_size' => 3,
                                'direct_generator' => [[
                                    'field' => 'title.trigram',
                                    'suggest_mode' => 'always'
                                ]],
                                'highlight' => [
                                    'pre_tag' => '<em>',
                                    'post_tag' => '</em>'
                                ]
                            ]
                        ]
                    ]
                ]
            ]);
        } catch (Exception $e) {
            $this->error($e);
        }
 
        $corrections = [];
        if (is_array($results['suggest']['simple_phrase'][0]['options']) && !empty($results['suggest']['simple_phrase'][0]['options'])) {
            $corrections = $results['suggest']['simple_phrase'][0]['options'];
        }
 
        return $corrections;
    }
 
    private function buildQuery($result, $relation, $query, $min_value, $multiply) {
        if (!empty($result)) {
            $ids = array_keys($result);
            if (empty($query['where'])) {
                $query['where'] = '';
            } else {
                $query['where'] .= ' AND ';
            }
 
            // Für Oracle splitten: maximal 1000 pro Liste
            $id_groups = [];
            foreach (array_chunk($ids, 999) as $id_group) {
                $id_groups[] = "$relation IN (" . implode(', ', $id_group) . ")";
            }
            $query['where'] .= '(' . implode(' OR ', $id_groups) . ')';
 
            $field2 = 'CASE ';
            foreach ($result as $id => $value) {
                $field2 .= "WHEN $relation = $id THEN " . (str_replace(',', '.', ($value - $min_value) * $multiply)) . " ";
            }
            $field2 .= 'ELSE 0 END AS score';
            $query['fields2'][] = $field2;
            $query['order'] = 'score DESC';
        } else { // No result, so construct an query with no results.
            unset($query['order']);
            unset($query['limit']);
            unset($query['bind']);
            $query['where'] = '1=0';
        }
        return $query;
    }
 
    public function setExtraQuery($query, $bind = []) {
        // Binds anwenden
        if (is_array($bind)) {
            // Zeichen, die von Elastic verwendet werden: + - && || ! ( ) { } [ ] ^ " ~ * ? : \ /
            $reserved_characters = preg_quote('+-&|!(){}[]^"~*?:\\');
            foreach ($bind as $key => $value) {
                $query = str_replace(
                    ":$key",
                    preg_replace_callback(
                        '/[' . $reserved_characters . ']/',
                        function ($matches) {
                            return '\\' . $matches[0];
                        },
                        $value
                    ),
                    $query
                );
            }
        }
 
        // Elastic Query generieren
        $Elastic_query = $query;
        $sub_queries = preg_split('/\s+(and|or)\s+/si', $query);
        foreach ($sub_queries as $sub_query) {
            if (preg_match('/(!?(extra\.)?[^ !=<>]+)(\s*(like|>=|<=|!=|=|>|<)\s*(.*?))?$/si', trim($sub_query, '() '), $matches)) {
                $param = $matches[1];
                $operator = mb_strtolower($matches[4]);
                $value = trim($matches[5], '\'"');
 
                // Feld darf nicht im Dokument gesetzt sein
                $exclude = false;
                if ($param[0] == '!') {
                    $exclude = true;
                    $param = substr($param, 1);
                }
 
                if ($exclude) {
                    // Ausnahme bilden
                    $replace = "-$param:*" . (sizeof($sub_queries) == 1 ? ' AND *' : '');
                } else {
                    // Elastic Syntax schreiben
                    $replace = "$param:";
                    if (!is_numeric($value)) {
                        if ($operator == 'like') {
                            $value = $this->prepareSearch(str_replace('%', '', $value), '', true);
                        } else {
                            $value = '"' . $value . '"';
                        }
                    }
 
                    // Vergleich bilden
                    switch ($operator) {
                        case '=':
                        case 'like':
                            $replace .= $value;
                            break;
                        case '!=':
                            $replace = "($param:* NOT $param:$value)";
                            break;
                        case '>':
                            $replace .= '{' . $value . ' TO *}';
                            break;
                        case '>=':
                            $replace .= '[' . $value . ' TO *]';
                            break;
                        case '<':
                            $replace .= '{* TO ' . $value . '}';
                            break;
                        case '<=':
                            $replace .= '[* TO ' . $value . ']';
                            break;
                        default:
                            $replace .= $value;
                    }
                }
 
                $Elastic_query = str_replace($matches[0], $replace, $Elastic_query);
            }
        }
 
        $this->extraQuery = $Elastic_query;
    }
 
    public function getConfig() {
        return $this->config;
    }
 
    public function clearCache() {
        foreach ($this->hosts as $host) {
            @file_get_contents($host . '_cache/clear');
        }
    }
 
    private function getTables($sites, $lang = '', &$relations = []) {
        $tables = '';
        if (!$lang) {
            $lang = $_REQUEST['lang'] ? $_REQUEST['lang'] : ($GLOBALS['site'] ? $GLOBALS['site']->language : null);
        }
        foreach ($sites as $site) {
            try {
                if (is_string($site)) {
                    $site = new Site($site);
                }
                if ($lang) {
                    $site->setLanguage($lang);
                }
            } catch (Exception $e) {
                // Mandant existiert nicht in dieser Sprache, ignorieren
                continue;
            }
            $table = strtolower($site->pageTable);
            $tables = $tables . ',' . $table;
            $relations[$table] = $site;
        }
        return ltrim($tables, ',');
    }
 
    private function error($e) {
        if ($this->throwException) {
            $class = get_class($e);
            throw new $class($e->getMessage(), $e->getCode());
        } else {
            $response = @json_decode($e->getMessage(), true);
            $messages = [];
            if ($response['error']) {
                if ($response['error']['root_cause']) {
                    foreach ($response['error']['root_cause'] as $cause) {
                        $messages[] = $cause['type'] . ' for index ' . $cause['index'];
                    }
                }
                if ($response['error']['failed_shards']) {
                    foreach ($response['error']['failed_shards'] as $shard) {
                        if (!empty($shard['reason']['caused_by'])) {
                            $messages[] = 'caused by ' . $shard['reason']['caused_by']['type'] . ' (reason: ' . $shard['reason']['caused_by']['reason'] . ')';
                        }
                    }
                }
            }
            if (empty($messages)) {
                $messages[] = $e->getMessage();
            }
            egotec_error_log('Elastic Exception thrown (' . get_class($e) . ', Code ' . $e->getCode() . '): ' . implode('; ', $messages));
            Ego_System::header($response['status'] ?? 400);
            exit;
        }
    }
}