diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml new file mode 100644 index 0000000..e427ab2 --- /dev/null +++ b/.github/workflows/generate-docs.yml @@ -0,0 +1,52 @@ +name: Generate and Deploy Documentation + +on: + push: + branches: + - main + +permissions: + actions: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + publish-docs: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 9.x + + - name: Install DocFX + run: dotnet tool install -g docfx + + - name: Add DocFX to PATH + run: echo "$HOME/.dotnet/tools" >> $GITHUB_PATH + + - name: Generate documentation + run: | + cd docs + docfx metadata + docfx build + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: '_site' + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/README.md b/README.md index ed7e1a6..c1cfae3 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,45 @@ -![WebExpress](https://raw.githubusercontent.com/ReneSchwarzer/WebExpress.Doc/main/assets/banner.png) +![WebExpress](https://raw.githubusercontent.com/ReneSchwarzer/WebExpress/main/assets/banner.png) # WebExpress -WebExpress is a lightweight web server optimized for use in low-performance environments (e.g. Rasperry PI). By providing +`WebExpress` is a lightweight web server optimized for use in low-performance environments (e.g. Rasperry PI). By providing a powerful plugin system and a comprehensive API, web applications can be easily and quickly integrated into a .net -language (e.g. C#). Some advantages of WebExpress are: +language (e.g. C#). Some advantages of `WebExpress` are: - It is easy to use. - It offers a variety of features and tools that can help you build and manage your website. - It is fast and efficient and can help you save time and money. - It is flexible and can be customized to meet your specific requirements. -The WebExpress family includes the following projects: +The `WebExpress` family includes the following projects: -- [WebExpress](https://github.com/ReneSchwarzer/WebExpress#readme) - The web server for WebExpress applications and the documentation. -- [WebExpress.WebCore](https://github.com/ReneSchwarzer/WebExpress.WebCore#readme) - The core for WebExpress applications. -- [WebExpress.WebUI](https://github.com/ReneSchwarzer/WebExpress.WebUI#readme) - Common templates and controls for WebExpress applications. -- [WebExpress.WebIndex](https://github.com/ReneSchwarzer/WebExpress.WebIndex#readme) - Reverse index for WebExpress applications. -- [WebExpress.WebApp](https://github.com/ReneSchwarzer/WebExpress.WebApp#readme) - Business application template for WebExpress applications. +- [WebExpress](https://github.com/ReneSchwarzer/WebExpress#readme) - The web server for `WebExpress` applications and the documentation. +- [WebExpress.WebCore](https://github.com/ReneSchwarzer/WebExpress.WebCore#readme) - The core for `WebExpress` applications. +- [WebExpress.WebUI](https://github.com/ReneSchwarzer/WebExpress.WebUI#readme) - Common templates and controls for `WebExpress` applications. +- [WebExpress.WebIndex](https://github.com/ReneSchwarzer/WebExpress.WebIndex#readme) - Reverse index for `WebExpress` applications. +- [WebExpress.WebApp](https://github.com/ReneSchwarzer/WebExpress.WebApp#readme) - Business application template for `WebExpress` applications. # WebExpress.WebIndex -WebExpress.WebIndex is part of the WebExpress family. The project provides a reverse index -to enable a quick and efficient search for data. The index can be filtered using the -wql (webexpress query language). Even though the reverse index is part of the WebExpress -family, it can also be used in other projects (outside of WebExpress). -For detailed information about WebIndex, see [concept](https://github.com/ReneSchwarzer/WebExpress.WebIndex/blob/main/doc/concept.md). +`WebExpress.WebIndex` is part of the WebExpress family. The project provides a reverse index to enable a quick and efficient search for data. The index can be filtered using the wql (webexpress query language). Even though the reverse index is part of the `WebExpress` family, it can also be used in other projects (outside of `WebExpress`). For detailed information about `WebIndex`, see [concept](https://github.com/ReneSchwarzer/WebExpress.WebIndex/blob/main/docs/concept.md). # Download The current binaries are available for download [here](https://github.com/ReneSchwarzer/WebExpress/releases). # Start -To get started with WebExpress, use the following links and tutorials. +If you're looking to get started with `WebExpress`, we would recommend using the following documentation. It can help you understand the platform. -- [installation guide](https://github.com/ReneSchwarzer/WebExpress/blob/main/doc/installation_guide.md) -- [development guide](https://github.com/ReneSchwarzer/WebExpress/blob/main/doc/development_guide.md) +- [Installation Guide](https://github.com/ReneSchwarzer/WebExpress/blob/main/doc/installation_guide.md) +- [Development Guide](https://github.com/ReneSchwarzer/WebExpress/blob/main/doc/development_guide.md) +- [WebExpress.WebCore API Documentation](https://reneschwarzer.github.io/WebExpress.WebCore/) +- [WebExpress.WebUI API Documentation](https://reneschwarzer.github.io/WebExpress.WebUI/) +- [WebExpress.WebApp API Documentation](https://reneschwarzer.github.io/WebExpress.WebApp/) +- [WebExpress.WebIndex API Documentation](https://reneschwarzer.github.io/WebExpress.WebIndex/) + +# Learning +The following tutorials illustrate the essential techniques of `WebExpress`. These tutorials are designed to assist you, as a developer, in understanding the various aspects of `WebExpress`. Each tutorial provides a detailed, step-by-step guide that you can work through using an example. If you’re interested in beginning the development of `WebExpress` components, we would recommend you to complete some of these tutorials. -## Tutorials - [HelloWorld](https://github.com/ReneSchwarzer/WebExpress.Tutorial.HelloWorld#readme) +- [WebApp](https://github.com/ReneSchwarzer/WebExpress.Tutorial.WebApp#readme) +- [WebIndex](https://github.com/ReneSchwarzer/WebExpress.Tutorial.WebIndex#readme) # Tags -#WebExpress #ReverseIndex \ No newline at end of file +#WebIndex #WebExpress #ReverseIndex #DotNet #NETCore \ No newline at end of file diff --git a/doc/concept.md b/doc/concept.md deleted file mode 100644 index 9fead52..0000000 --- a/doc/concept.md +++ /dev/null @@ -1,989 +0,0 @@ -ο»Ώ![WebExpress](https://raw.githubusercontent.com/ReneSchwarzer/WebExpress.Doc/main/assets/banner.png) - -# WebExpress.WebIndex -The index model provides a reverse index to enable fast and efficient searching. A reverse index can significantly speed up access -to the data. However, creating and storing a reverse index requires additional storage space and processing time. The storage -requirement increases, especially with large amounts of data, which can be important. Therefore, it is important to weigh the -pros and cons to achieve the best possible performance. The reverse index offers the added value of a fast and resource-saving -full-text search and accepts the costs mentioned. The full-text search in `WebExpress` supports the following search options: - -- Word search -- Wildcard search -- Phrase search (exact word sequence) -- Proximity search -- Fuzzy search - -The indexing process begins with the analysis of documents, where the documents are broken down into smaller units, usually words -or phrases. These broken-down units are then converted into normalized tokens. Normalization can take various forms, such as converting -all letters to lowercase, removing punctuation, or reducing words to their stem. In addition, stop words are removed. Stop words are -frequently occurring words like "and", "is", "in", which typically do not provide added value for the search. These words are filtered -out to improve efficiency and reduce storage requirements. In many languages, words can appear in different forms that all refer to the -same concept. Therefore, techniques such as stemming, or lemmatization are often applied to reduce different forms of a word to a common -representation. These additional steps help improve the accuracy and relevance of search results and help keep the index compact and -manageable. The normalized tokens are stored in a reverse index. This index is structured so that it lists the documents in which each -token appears for each token. In addition to the tokens, more information is stored, such as the frequency of the tokens or the position -of the tokens in the document. During the search process, the search words are tokenized and normalized in the same way, and then each -token is looked up in the reverse index. The documents or positions found in the lists of all tokens are the search results. This method -allows for a fast and efficient search, as the time-consuming part is carried out in advance during the indexing process and the actual -search consists only of quick lookup operations in the reverse index. - -``` - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” indexing - β”‚ document β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ - β–Ό - β”Œβ”€β”€β”€β”€β”€β”€β”€β” searching β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” ╔══════════╗ - β”‚ query β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€>β”‚ tokenizer β”œβ”€β”€β”€β”€β”€β”€>β”‚ stemming, lemmatization & stoppword filter pipe β”œβ”€β”€β”€β”€β”€β”€>β•‘ WebIndex β•‘ - β””β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β•šβ•β•β•β•β•β•β•β•β•β•β• - β–² results β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -Stemming and lemmatization are text preprocessing techniques in natural language processing (NLP). They reduce the inflected forms of -words in a text dataset to a common root form or dictionary form, also known as a "lemma" in computational linguistics. - -Stemming usually refers to a crude heuristic process that trims the ends of words in the hope of achieving this goal mostly correctly, and -often involves removing derivational affixes. Stemming algorithms attempt to find the common roots of different inflections by cutting -off the endings or beginnings of the word. - -Lemmatization usually refers to doing things correctly, using a vocabulary and morphological analysis of words, usually with the aim of -removing only inflectional endings and returning the base or dictionary form of a word, known as a lemma. Unlike stemming, which -operates on a single word without knowledge of the context, lemmatization can distinguish between words that have different meanings -depending on the part of speech. - -These techniques are particularly useful in information search systems such as search engines, where users can submit a query using -one word (e.g., "meditate") but expect results that use any inflected form of the word (e.g. "meditated", "meditation", etc). - -In this instance, indexing is performed on two documents by executing a series of operations: tokenization, normalization, and -stop-word removal. The outcome of these operations is a multi-dimensional table, which serves as a representation of the reverse index. - -``` - β”Œdocument a────────────────────────────────────────┐ β”Œdocument b────────────────────────────────────────┐ - β”‚ No, fine, no , good, fine, good. You know Marty, β”‚ β”‚ Thanks a lot, kid. Now, of course not, Biff, now,β”‚ - β”‚ you look so familiar, do I know your mother? Hey β”‚ β”‚ I wouldn't want that to happen. I'm gonna ram β”‚ - β”‚ man, the dance is over. Unless you know someone β”‚ β”‚ him. Well, Marty, I want to thank you for all β”‚ - β”‚ else who could play the guitar. Who's are these? β”‚ β”‚ your good advice, I'll never forget it. Doc, β”‚ - β”‚ Maybe you were adopted. β”‚ β”‚ look, all we need is a little plutonium. β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ β”‚ - β”‚ β”‚ - β–Ό β–Ό - β”Œnormalized document a─────────────────────────────┐ β”Œnormalized document b─────────────────────────────┐ - β”‚ no fine no good fine good you know marty β”‚ β”‚ thank a lot kid now of course not biff now β”‚ - β”‚ you look so familiar do i know your mother hey β”‚ β”‚ i would not want that to happen i am gonna ram β”‚ - β”‚ man the dance is over unless you know someone β”‚ β”‚ him well marty i want to thank you for all β”‚ - β”‚ else who could play the guitar who is are these β”‚ β”‚ your good advice i will never forget it doc β”‚ - β”‚ maybe you were adopted β”‚ β”‚ look all we need is a little plutonium β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ β”‚ - β”‚ β”‚ - β–Ό β–Ό - β”Œstopword cleaned document a───────────────────────┐ β”Œstopword cleaned document b───────────────────────┐ - β”‚ fine good fine good know marty β”‚ β”‚ thank lot kid course biff β”‚ - β”‚ look familiar know mother β”‚ β”‚ would want happen gonna ram β”‚ - β”‚ dance unless know someone β”‚ β”‚ well marty want thank β”‚ - β”‚ else could play guitar these β”‚ β”‚ good advice never forget doc β”‚ - β”‚ maybe adopted β”‚ β”‚ look need little plutonium β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ β”‚ - β”‚ β”‚ - └───────────────┐ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β–Ό β–Ό - β•”WebIndex═══════════════════════════════════════╗ - β•‘ Term β”‚ Fequency β”‚ Documnet β”‚ Position β•‘ - ║═══════════════════════════════════════════════║ - β•‘ adopted β”‚ 1 β”‚ a β”‚ 211 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ advice β”‚ 1 β”‚ b β”‚ 153 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ biff β”‚ 1 β”‚ b β”‚ 40 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ could β”‚ 1 β”‚ a β”‚ 156 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ course β”‚ 1 β”‚ b β”‚ 28 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ dance β”‚ 1 β”‚ a β”‚ 108 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ doc β”‚ 1 β”‚ b β”‚ 183 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ else β”‚ 1 β”‚ a β”‚ 147 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ familiar β”‚ 1 β”‚ a β”‚ 62 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ fine β”‚ 2 β”‚ a β”‚ 5 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ forget β”‚ 1 β”‚ b β”‚ 22 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ gonna β”‚ 1 β”‚ b β”‚ 87 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ good β”‚ 3 β”‚ a β”‚ 16, 28 β•‘ - β•‘ β”‚ β”‚ b β”‚ 148 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ guitar β”‚ 1 β”‚ a β”‚ 171 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ happen β”‚ 1 β”‚ b β”‚ 75 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ kid β”‚ 1 β”‚ b β”‚ 15 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ know β”‚ 3 β”‚ a β”‚ 38, 77, 134 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ little β”‚ 1 β”‚ b β”‚ 211 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ look β”‚ 2 β”‚ a β”‚ 54 β•‘ - β•‘ β”‚ β”‚ b β”‚ 188 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ lot β”‚ 1 β”‚ b β”‚ 10 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ marty β”‚ 2 β”‚ a β”‚ 43 β•‘ - β•‘ β”‚ β”‚ b β”‚ 108 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ maybe β”‚ 1 β”‚ a β”‚ 196 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ mother β”‚ 1 β”‚ a β”‚ 87 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ need β”‚ 1 β”‚ b β”‚ 201 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ never β”‚ 1 β”‚ b β”‚ 166 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ play β”‚ 1 β”‚ a β”‚ 162 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ plutonium β”‚ 1 β”‚ b β”‚ 218 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ ram β”‚ 1 β”‚ b β”‚ 93 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ someone β”‚ 1 β”‚ a β”‚ 139 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ thank β”‚ 2 β”‚ b β”‚ 0, 125 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ these β”‚ 1 β”‚ a β”‚ 189 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ unless β”‚ 1 β”‚ a β”‚ 123 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ want β”‚ 2 β”‚ b β”‚ 62, 117 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ well β”‚ 1 β”‚ b β”‚ 102 β•‘ - ║───────────┼──────────┼──────────┼─────────────║ - β•‘ would β”‚ 1 β”‚ b β”‚ 53 β•‘ - β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• - β–² - β”‚ - β”‚ - β”Œstop word cleaned query──┐ - β”‚ marty play guitar β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β–² - β”‚ - β”‚ - β”Œnormalized query─────────┐ - β”‚ marty play the guitar β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β–² - β”‚ - β”‚ - β”Œquery───────┴─────────────┐ - β”‚ 'Marty play the guitar.' β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -`WebIndex` is an efficient system that combines document store and reverse indices to support a variety of search options. The -`IndexDocumentStore` stores all instances of a document for quick access, regardless of other persistent storage forms such as -databases. On the other hand, the reverse index is created for each field `IndexField` of a document unless it is marked with -`IndexIgnore`. The field contents are tokenized, normalized, and filtered to create the terms of the reverse index. Each term -in the reverse index is associated with a posting that contains the IDs of the document instances that contain the term. The -position where the term was found within the attribute value is stored in the position. There can be multiple positions for each -posting. When searching for one or more terms, the IDs of the instances and their positions within the attribute values can be -determined. - -``` - ╔═══════════════════════════════════════ IndexManager ═╗ - β•‘ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β•‘ - β•‘ β”‚ WebIndex β”‚ β•‘ - β•‘ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β•‘ - β•‘ β”‚ 1 β•‘ - β•‘ β”‚ ╔══════ IndexDocumentStore ═╗ β•‘ - β•‘ β”‚ * β•‘ β•‘ β•‘ - β•‘ β”Œβ”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β” 1 β•‘ * β”Œβ”€β”€β”€β”€β”€β”€β” β•‘ β•‘ - β•‘ β”‚ IndexDocument β”œβ”€β”€β”€β”€β”€β”€β”€β”€ Item β”‚ β•‘ β•‘ - β•‘ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β””β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ - β•‘ β”‚ 1 β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• β•‘ - β•‘ β”‚ β•‘ - β•‘ β”‚ * β•‘ - β•‘ β”Œβ”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β” β•‘ - β•‘ β”‚ IndexField β”‚ β•‘ - β•‘ β””β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”˜ β•‘ - β•‘ β”‚ 1 β•‘ - β•‘ ╔══════│═════ IndexReverse ═╗ β•‘ - β•‘ β•‘ β”‚ * β•‘ β•‘ - β•‘ β•‘ β”Œβ”€β”€β”€β”΄β”€β”€β” β•‘ β•‘ - β•‘ β•‘ β”‚ Term β”‚ β•‘ β•‘ - β•‘ β•‘ β””β”€β”€β”€β”¬β”€β”€β”˜ β•‘ β•‘ - β•‘ β•‘ β”‚ 1 β•‘ β•‘ - β•‘ β•‘ β”‚ β•‘ β•‘ - β•‘ β•‘ β”‚ * β•‘ β•‘ - β•‘ β•‘ β”Œβ”€β”€β”€β”€β”΄β”€β”€β”€β”€β” β•‘ β•‘ - β•‘ β•‘ β”‚ Posting β”‚ β•‘ β•‘ - β•‘ β•‘ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”˜ β•‘ β•‘ - β•‘ β•‘ β”‚ 1 β•‘ β•‘ - β•‘ β•‘ β”‚ β•‘ β•‘ - β•‘ β•‘ β”‚ * β•‘ β•‘ - β•‘ β•‘ β”Œβ”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β” β•‘ β•‘ - β•‘ β•‘ β”‚ Position β”‚ β•‘ β•‘ - β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ - β•‘ β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• β•‘ - β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• -``` - -# IndexManager -The index manager is a central component of the `WebIndex` system and serves as the primary interface for interacting with the indexing -functions. It is responsible for managing the various `IndexDocuments` that are created in `WebIndex`. Each `IndexDocument` represents -a collection of documents that need to be indexed, and the index manager ensures that these documents are indexed correctly and -efficiently. In addition, the index manager provides functions for adding, updating, and deleting documents in the index. It also enables -the execution of search queries on the index and returns the corresponding results. Finally, the index manager provides high control over -the indexing process by allowing certain fields to be excluded from indexing or determining whether the index should be created in main -memory or persistently in the file system. An `IndexDocument` created in main memory enables faster indexing and searching. However, the -number of objects it can support is limited and depends on the size of the available main memory. Therefore, it is important to weigh the -pros and cons and choose the best solution for the specific requirements. The following diagram serves as a guide to estimate the -performance and resources required when using the file-based approach: - -``` - TIME CHART >01:50 - [h] β–² [ms] β–“ - β”‚ β–“ - 0:24 β”Ό 12 β‰ˆ - 0:23 ─ β–“ - 0:22 β”Ό 11 β–“ - 0:21 ─ β–“ - 0:20 β”Ό 10 β–“ - 0:19 ─ β–“ - 0:18 β”Ό 9 β–“ - 0:17 ─ β–“ - 0:16 β”Ό 8 β–“ - 0:15 ─ β–“ - 0:14 β”Ό 7 β–“ - 0:13 ─ β–“ - 0:12 β”Ό 6 0:11 β–“ - 0:11 ─ 0:10 β–“ β–“>5 - 0:10 β”Ό 5 β–“ β–“ β–“β–‘ - 0:09 ─ 0:08 β–“ β–“ β–“β–‘ - 0:08 β”Ό 4 0:07 β–“ β–“ β–“ β–“β–‘ - 0:06 ─ 0:06 β–“ β–“ β–“ β–“ β–“β–‘ - 0:06 β”Ό 3 0:05 β–“ β–“ β–“ β–“ β–“ β–“β–‘ - 0:05 ─ 0:04 β–“ β–“ β–“ β–“4 β–“ β–“ β–“β–‘ - 0:04 β”Ό 2 0:03 β–“ β–“ β–“ β–“ β–“β–‘ β–“3 β–“ β–“β–‘ - 0:03 ─ 2 0:02 β–“ β–“ β–“2 β–“2 β–“2 β–“β–‘ β–“β–‘ β–“2 β–“β–‘ - 0:02 β”Ό 1 0:01β–‘ β–“1 β–“1 β–“1 β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ - 0:01 ─ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ - └──────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬───~~~───┬─────► - 10,000 20,000 30,000 40,000 50,000 60,000 70,000 80,000 90,000 100,000 1,000,000 [items] - - β–“ - time required to reindex in [h] - β–‘ - time required to retrieve in [ms] -``` - -``` - STORAGE SPACE CHART >5,000 - [MB] β–² β–’ - β”‚ 573 β‰ˆ - 550 ─ β–’ β–’ - β”‚ 516 β–’ β–’ - 500 ─ β–’ β–’ 1,525β–’ - β”‚ 460 β–’ β–’ β–ˆβ–’ - 450 ─ β–’ β–’ β–’ β‰ˆβ–’ - β”‚ β–’ β–’ β–’ β–ˆβ–’ - 400 ─ 403 β–’ β–’ β–’ β–ˆβ–’ - β”‚ β–’ β–’ β–’ β–’ β–ˆβ–’ - 350 ─ 345 β–’ β–’ β–’ β–’ β–ˆβ–’ - β”‚ β–’ β–’ β–’ β–’ β–’ β–ˆβ–’ - 300 ─ 291 β–’ β–’ β–’ β–’ β–’ β–ˆβ–’ - β”‚ β–’ β–’ β–’ β–’ β–’ β–’ β–ˆβ–’ - 250 ─ β–’ β–’ β–’ β–’ β–’ β–’ β–ˆβ–’ - β”‚ 234 β–’ β–’ β–’ β–’ β–’ β–’ β–ˆβ–’ - 200 ─ β–’ β–’ β–’ β–’ β–’ β–’ β–’ β–ˆβ–’ - β”‚ 178 β–’ β–’ β–’ β–’ β–’ β–’ β–’ β–ˆβ–’ - 150 ─ β–’ β–’ β–’ β–’ β–’ β–’ β–’ 152β–’ β–ˆβ–’ - β”‚ 121 β–’ β–’ β–’ β–’ β–’ 122β–’ 137β–’ β–ˆβ–’ β–ˆβ–’ - 100 ─ β–’ β–’ β–’ β–’ β–’ 107β–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ - β”‚ 65 β–’ β–’ 61β–’ 76β–’ 91β–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ - 50 ─ 15β–’ 30β–’ 45β–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ - β”‚ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ - └──────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬───~~~───┬─────► - 10,000 20,000 30,000 40,000 50,000 60,000 70,000 80,000 90,000 100,000 1,000,000 [items] - - β–ˆ - payload in [MB], calculation: payload = item count * (100 wrods * 15 character + 99 spaces) / 1024 / 1024 - β–’ - storage space required in [MB] -``` - -It should be noted that the actual execution times and memory consumption depend heavily on the platform on which WebExpress-WebIndex -is operated. In addition, the vocabulary, the content of the documents and the number of IndexFields have a significant impact on -the performance of the system. The measured values apply to the following conditions: The document contains a field with 100 words -each consisting of 15 characters. The words come from a vocabulary of 20,000 words. The series of measurements was created on the -following system: - -- OS: Windows 11 64bit -- System: Intel NUC-13 -- Processor: Intel i7-1360P -- RAM: 64,0 GB (SODIMM 3200 MHz) RAM -- HDD: Samsung SSD 990 Pro 2TB - -The `IndexManager` class offers a variety of functions for managing and optimizing indexed data. Here are some of the main -methods and properties of this class: - -- `IIndexContext Context`: The context of the index. -- `void Initialization(IIndexContext context)`: Initialization of the IndexManager. -- `void Register(IIndexPipeStage pipeStage)`: Registers a pipe state for processing the tokens. -- `void ReIndex(IEnumerable items)`: Reindexing of the index. -- `Task ReIndexAsync(IEnumerable items, IProgress progress, CancellationToken token)`: Performs an asynchronous reindexing of a collection of index items. -- `void Create(CultureInfo culture, IndexType type)`: Registers a data type in the index. -- `void Close()`: Closes the index file of type T. -- `Task CloseAsync()`: Asynchronously closes the index file of type T. -- `void Drop()`: Drops all index documents of type T. -- `Task DropAsync()`: Asynchronously drops all index documents of type T. -- `void Insert(T item)`: Adds an item to the index. -- `Task InsertAsync(T item)`: Performs an asynchronous addition of an item in the index. -- `void Update(T item)`: Updates an item in the index. -- `Task UpdateAsync(T item)`: Performs an asynchronous update of an item in the index. -- `void Delete(T item)`: Removes an item from the index. -- `uint Count()`: Counts the number of items of the index. -- `Task CountAsync()`: Performs an asynchronous determination of the number of elements. -- `Task DeleteAsync(T item)`: Removes an item from the index asynchronously. -- `void Clear()`: Removed all data from the index. -- `Task ClearAsync()`: Removed all data from the index asynchronously. -- `IWqlStatement Retrieve(string wql)`: Executes a WQL statement. -- `Task> RetrieveAsync(string wql)`: Executes a wql statement asynchronously. -- `IEnumerable All()`: Returns all documents from the index. -- `IIndexDocument GetIndexDocument()`: Returns an index type based on its type. -- `void Dispose()`: Disposes of the resources used by the current instance. - -## IndexDocument -An `IndexDocument` representing a class that implements the `IIndexItem` interface. Each `IndexDocument` contains a collection of fields -that hold the data to be indexed. These fields can contain various types of data, such as text, numbers, or dates. During the indexing -process, the data in these fields are analyzed and tokenized, then stored in the reverse index. In addition to the reverse index, an -`IndexDocument` also includes a document store for quick access to the existing instances. When a search query is made for one or more -terms, the IDs of the instances that match the terms are identified in the reverse index and supplemented with the corresponding instances -in the document store and returned to the searcher. - -## IndexField -An `IndexField` is a property (C# property) in an index document that can accommodate various types of values, such as text, numbers, or -other data. Each field is stored in a reverse index, which converts the field values into terms and associates them with document IDs and -positional data in the reverse index. The name and type of the field are essential pieces of information used during indexing and searching. -If a field is marked with the IndexIgnore attribute, it will be excluded from the indexing process. - -``` - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ <> β”‚ - β”‚ IndexDocument β”œβ”€β”€β”€β”€β”€β”€β”€β”€ IndexDocumentStore β”‚ β”‚ IIndexItem β”‚ - β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ β–² - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” Β¦ - β”‚ Property 1 β”‚ Property 2 β”‚ Property … β”‚ Property n β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”Œβ”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β” β”‚ MyIndexItem β”‚ - β”‚ IndexField 1 β”‚ β”‚ IndexField 2 β”‚ β”‚ IndexField … β”‚ β”‚ IndexField n β”‚ <--> β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ Property 1 β”‚ - β”‚ β”‚ β”‚ β”‚ β”‚ Property 2 β”‚ - β”Œβ”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ Property … β”‚ - β”‚ IndexReverse 1 β”‚ β”‚ IndexReverse 2 β”‚ β”‚ IndexReverse … β”‚ β”‚ IndexReverse n β”‚ β”‚ Property n β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` -## IndexSchema -The index schema file contains important metadata, which provides detailed information about the structure and characteristics of -the provide indexes. In addition, the file contains a precise object description of the document captured by the index and is -managed. Furthermore, the JSON (JavaScript Object Notation) format is used for the index schema file and the have the extension `*.ws`. - -## IndexStore -In a filesystem where the `WebIndex` is stored, a process is carried out where an inverted index is created for each field. These indexes are -stored as files with the `.wri` extension. In parallel, a special storage area known as the document store -`.wds` is set up for each document. In this storage area, the document’s data is redundantly stored to enable quick access. -The structure of these files follows a uniform format that is divided into various segments. Each of these segments is identifiable by a -unique address and has a specific length. There is the option to specify whether a particular segment should be stored in the cache. If a -segment is no longer needed, it can be removed from the main memory. These features contribute to efficient use of storage and improve -the system performance. - -``` - β•”Header═════════╗ - 3 Byte β•‘ "wds" | "wri" β•‘ identification (magic number) of the file `wds` for IndexDocumentStore and `wri` for IndexReverse - 1 Byte β•‘ Version β•‘ the file version - β• Allocator══════╣ - 8 Byte β•‘ NextFreeAddr β•‘ the next free address - 8 Byte β•‘ FreeListAddr β•‘ the address to the free list - β• Statistic══════╣ - 4 Byte β•‘ Count β•‘ the number of elements in the file - β• Body═══════════╣ - β•‘ β•‘ a variable memory area in which the data is stored - β•š~~~~~~~~~~~~~~~╝ -``` - -Unused memory areas in the file are represented by the `Free` segment, which is located in the body area variable and forms a linked list. The -`Allocator` points to the first element of this list. - -``` - β•”Free═══════════╗ - 8 Byte β•‘ SuccessorAddr β•‘ pointer to the address of the next element of a sorted list or 0 if there is no element - β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• -``` - -When new documents are indexed, the new segments are accommodated in a free storage area in the file. Initially, this is the end of the -file. In fragmented files, where segments have already been deleted, the freed storage areas are reused. The `Allocator` in the header always -points to the next free storage space with `NextFreeAddr`. - -### Example alloc -In this example, segments 2 and 3 are successively added. It is important to note that segment 1 already exists. - -``` - initial add segment 2 add segment 3 - - 0 β•”Header═════════╗ β•”Header═════════╗ β•”Header═════════╗ - β•‘ "wds" | "wri" β•‘ β•‘ "wds" | "wri" β•‘ β•‘ "wds" | "wri" β•‘ - 3 β• Allocator══════╣ β• Allocator══════╣ β• Allocator══════╣ - β•‘ 2 β•‘ β•‘ 3 β•‘ β•‘ 4 β•‘ - β•‘ 0 β•‘ β•‘ 0 β•‘ β•‘ 0 β•‘ -19 β• Statistic══════╣ β• Statistic══════╣ β• Statistic══════╣ - β•‘ 1 β•‘ β•‘ 2 β•‘ β•‘ 3 β•‘ -23 β• Body═══════════╣ β• Body═══════════╣ β• Body═══════════╣ - β•‘ β”ŒSeg: 1┐ β•‘ => β•‘ β”ŒSeg: 1┐ β•‘ => β•‘ β”ŒSeg: 1┐ β•‘ - β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ - β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ - β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• β•‘ β”ŒSeg: 2┐ β•‘ β•‘ β”ŒSeg: 2┐ β•‘ - β•‘ β”‚ + β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ - β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ - β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• β•‘ β”ŒSeg: 3┐ β•‘ - β•‘ β”‚ + β”‚ β•‘ - β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ - β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• -``` - -Free spaces are stored in a linked list that represents the free segments in the file. These can be reused to store new data. Unused -Segments are replaced by the `free` segment. - -### Example Free -In this example, segments 2, 1 and 4 are sequentially released and consolidated. - -``` - initial remove segment 2 remove segment 1 remove segment 4 - - 0 β•”Header═════════╗ β•”Header═════════╗ β•”Header═════════╗ β•”Header═════════╗ - β•‘ "wds" | "wri" β•‘ β•‘ "wds" | "wri" β•‘ β•‘ "wds" | "wri" β•‘ β•‘ "wds" | "wri" β•‘ - 3 β• Allocator══════╣ β• Allocator══════╣ β• Allocator══════╣ β• Allocator══════╣ - β•‘ 5 β•‘ β•‘ 5 β•‘ β•‘ 5 β•‘ β•‘ 5 β•‘ - β•‘ 0 β•‘ β•‘ 2 ║─┐ β•‘ 1 ║─┐ β•‘ 4 ║───┐ -19 β• Statistic══════╣ β• Statistic══════╣ β”‚ β• Statistic══════╣ β”‚ β• Statistic══════╣ β”‚ - β•‘ 4 β•‘ β•‘ 3 β•‘ β”‚ β•‘ 2 β•‘ β”‚ β•‘ 1 β•‘ β”‚ -23 β• Body═══════════╣ β• Body═══════════╣ β”‚ β• Body═══════════╣ β”‚ β• Body═══════════╣ β”‚ - β•‘ β”ŒSeg: 1┐ β•‘ => β•‘ β”ŒSeg: 1┐ β•‘ β”‚ => β•‘ β”ŒFree: 1┐ β•‘β—„β”˜ => β•‘ β”ŒFree: 1┐ ║◄─┐│ - β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β”‚ β•‘ β”‚ X β”‚ ║─┐ β•‘ β”‚ β”‚ ║─┐││ - β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚β”‚β”‚ - β•‘ β”ŒSeg: 2┐ β•‘ β•‘ β”ŒFree: 2┐ β•‘β—„β”˜ β•‘ β”ŒFree: 2┐ β•‘β—„β”˜ β•‘ β”ŒFree: 2┐ β•‘β—„β”˜β”‚β”‚ - β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ X β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β”‚β”‚ - β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚β”‚ - β•‘ β”ŒSeg: 3┐ β•‘ β•‘ β”ŒSeg: 3┐ β•‘ β•‘ β”ŒSeg: 3┐ β•‘ β•‘ β”ŒSeg: 3┐ β•‘ β”‚β”‚ - β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β”‚β”‚ - β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚β”‚ - β•‘ β”ŒSeg: 4┐ β•‘ β•‘ β”ŒSeg: 4┐ β•‘ β•‘ β”ŒSeg: 4┐ β•‘ β•‘ β”ŒFree: 4┐ β•‘β—„β”€β”Όβ”˜ - β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ X β”‚ β•‘β”€β”€β”˜ - β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ - β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• -``` - -The repurposing of unused segments reduces the space requirements of files, particularly for highly fluctuating index files. This practice -not only optimizes storage but also enhances the overall performance and efficiency of data management systems. It is especially beneficial -in environments where data is frequently updated or deleted, leading to a high turnover of index files. By reusing these segments, the system -can maintain optimal performance while minimizing the need for additional storage space. - -### Example realloc -This example reallocates a segment using the available free space. Since the size of the segments is fixed, it is irrelevant which free one -Segment is reused. For efficiency reasons, the first element from the list of free segments is always used. - -``` - initial realloc segment 4 - - 0 β•”Header═════════╗ β•”Header═════════╗ - β•‘ "wds" | "wri" β•‘ β•‘ "wds" | "wri" β•‘ - 3 β• Allocator══════╣ β• Allocator══════╣ - β•‘ 5 β•‘ β•‘ 5 β•‘ - β•‘ 4 ║───┐ β•‘ 1 ║─┐ -19 β• Statistic══════╣ β”‚ β• Statistic══════╣ β”‚ - β•‘ 1 β•‘ β”‚ β•‘ 2 β•‘ β”‚ -23 β• Body═══════════╣ β”‚ β• Body═══════════╣ β”‚ - β•‘ β”ŒFree: 1┐ ║◄─┐│=> β•‘ β”ŒSeg: 1─┐ β•‘β—„β”˜ - β•‘ β”‚ β”‚ ║─┐││ β•‘ β”‚ β”‚ ║─┐ - β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚β”‚β”‚ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚ - β•‘ β”ŒFree: 2┐ β•‘β—„β”˜β”‚β”‚ β•‘ β”ŒFree: 2┐ β•‘β—„β”˜ - β•‘ β”‚ β”‚ β•‘ β”‚β”‚ β•‘ β”‚ β”‚ β•‘ - β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚β”‚ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ - β•‘ β”ŒSeg: 3┐ β•‘ β”‚β”‚ β•‘ β”ŒSeg: 3┐ β•‘ - β•‘ β”‚ β”‚ β•‘ β”‚β”‚ β•‘ β”‚ β”‚ β•‘ - β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚β”‚ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ - β•‘ β”ŒFree: 4┐ β•‘β—„β”€β”Όβ”˜ β•‘ β”ŒSeg: 4┐ β•‘ - β•‘ β”‚ β”‚ β•‘β”€β”€β”˜ β•‘ β”‚ x β”‚ β•‘ - β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ - β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• -``` - -### Caching -Caching is an efficient technique for optimizing data access by enabling fast access to frequently used data and simultaneously reducing -the load on the file system. It stores frequently used data in memory, which speeds up access to this data as it does not have to be -retrieved from the hard drive again. For write accesses, the data is first written to the read cache. The read cache uses a hash map to -allow random access to the cached segments. Each cached segment has a defined lifetime. If this has expired, the segments are removed from -the read cache, unless they have been marked as immortal via the `SegmentCached` attribute. The maximum size of the read cache can be -determined using the `IndexStorageReadBuffer.MaxCachedSegments` parameter. The size influences all `IndexStore` files and can be changed -during operation. However, it should be noted that if the size of memory already allocated will not be released. - - -## IndexDocumentStore -A `IndexDocumentStore` is a data structure in which each key is associated with a value. This allows efficient retrieval and retrieval of data -based on the key. The document store plays a crucial role in improving the efficiency of queries by enabling direct access to the document -instances that contain the desired terms. The internal structure of the document store: - -``` - β•”Header═════════╗ - 3 Byte β•‘ "wds" β•‘ identification (magic number) of the file - 1 Byte β•‘ Version β•‘ the file version - β• Allocator══════╣ - 8 Byte β•‘ NextFreeAddr β•‘ the next free address - 8 Byte β•‘ FreeItemAddr β•‘ the address to the list with free item node segments - 8 Byte β•‘ FreeChunkAddr β•‘ the address to the list with free chunk node segments - β• Statistic══════╣ - 4 Byte β•‘ Count β•‘ the number of terms in the file - β• Body═══════════╣ - β•‘ HashMap β•‘ a hash map in which the data is stored - β•š~~~~~~~~~~~~~~~╝ -``` - -Access to the document instances is done via a HashMap, where the id serves as the key. - -``` - β•”HashMap════════╗ - 4 Byte β•‘ BucketCount β•‘ number of buckets (the next prime number of capacity) - β• Data═══════════╣ - β•‘ Bucket 0 β•‘ pointer to the address of the first element of a sorted list that has the same hash values (collisions) or 0 if there is no element - n * β•‘ Bucket 1 β•‘ - 8 Byte β•‘~~~~~~~~~~~~~~~β•‘ - β•‘~~~~~~~~~~~~~~~β•‘ - β•‘ Bucket n β•‘ - β•‘ β•‘ a variable memory area in which the data is stored - β•š~~~~~~~~~~~~~~~╝ -``` - -The document instances are stored in one or more segments. The size of the segment is fixed and the number of segments per record is -determined by the size of the compressed document instance. The segment is stored in the variable storage area. - -``` - β•”Item═══════════╗ - 16 Byte β•‘ Id β•‘ guid of the document item - 4 Byte β•‘ Length β•‘ size of the DataChunk in bytes -256 Byte β•‘ DataChunk β•‘ a memory area in which a part of the element is stored (gzip compressed) - 8 Byte β•‘ NextChunkAddr β•‘ pointer to the address of the next chunk element of a list or 0 if there is no element - 8 Byte β•‘ SuccessorAddr β•‘ pointer to the address of the next bucket element of a sorted list or 0 if there is no element - β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• -``` - -If the data is larger than what can fit in a chunk, additional chunks are created and the data is divided among them. The last chunk -may not be completely filled. All chunks are linked in an ordered list. -``` - β•”Chunk══════════╗ - 4 Byte β•‘ Length β•‘ size of the DataChunk in bytes -256 Byte β•‘ DataChunk β•‘ a memory area in which a part of the element is stored (gzip compressed) - 8 Byte β•‘ NextChunkAddr β•‘ pointer to the address of the next chunk element of a list or 0 if there is no element - β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• -``` - -**Add**: The add function in the `IndexDocumentStore` is designed to permanently store the entire document. To efficiently utilize -storage space and minimize storage requirements, the object is serialized into a JSON format and subsequently compressed. -This process ensures efficient use of storage space without compromising the integrity or accessibility of the data. Furthermore, -this method allows for faster data transmission and enhances the overall performance of the `IndexDocumentStore`. Access to the -original data can be obtained at any time through decompression and deserialization. -``` - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ start β”‚ - β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ if !contains(id) β”‚ look up document id in hash map - β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ gzip(data) β”‚ gzip the data - β”‚ β”‚ β”‚ add item β”‚ adding an items segment - β”‚ β”‚ └─────────────────────────── - β”‚ β”‚ else β”‚ - β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ throw ArgumentException β”‚ - β”‚ β”‚ └─────────────────────────── - β”‚ β”‚ end if β”‚ - β”‚ └───────────────────────────── - β”‚ end β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -**Update**: The update process consists of a combination of delete and add operations. If the data is of the same size, the existing item -segment is reused. Otherwise, a new item segment is created and used. This approach optimizes storage usage and enhances -system efficiency by avoiding unnecessary storage allocations. - -``` - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ start β”‚ - β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ if contains(id) β”‚ look up document id in hash map - β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ delete β”‚ delete item - β”‚ β”‚ β”‚ gzip(data) β”‚ gzip the data - β”‚ β”‚ β”‚ delete item β”‚ remove the existing item segment - β”‚ β”‚ β”‚ add item β”‚ adding the updated item segment - β”‚ β”‚ └─────────────────────────── - β”‚ β”‚ else β”‚ - β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ add β”‚ add item - β”‚ β”‚ └─────────────────────────── - β”‚ β”‚ end if β”‚ - β”‚ └───────────────────────────── - β”‚ end β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -**Remove**: Documents that are no longer needed can be securely removed from the document storage by using the delete function. This -ensures efficient use of storage and keeps the document storage tidy and well-organized. - -``` - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ start β”‚ - β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ if !contains(id) β”‚ look up document id in hash map - β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ delete item β”‚ remove the existing item segment - β”‚ β”‚ └─────────────────────────── - β”‚ β”‚ else β”‚ - β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ throw ArgumentException β”‚ - β”‚ β”‚ └─────────────────────────── - β”‚ β”‚ end if β”‚ - β”‚ └───────────────────────────── - β”‚ end β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -### IndexReverse -A reverse index is a specialized type of index that allows access to data in reverse order. In the context of the `WebIndex`, the reverse -index is used for efficient searching of terms. These terms are derived from the associated fields, and their values are broken down -into tokens, normalized, filtered, and stored in a search tree for fast retrieval. - -Unlike the general definition of the header, `IndexReverse` has a modified allocator. A separate list is kept for each segment type -in which the free segments are saved. For faster storage and reuse, insertion and removal only ever occurs at the beginning of -the list. When there is a new memory request, a free segment can be reused without any search effort. Merging segments is not necessary. - -``` - β•”Header═══════════╗ - 3 Byte β•‘ "wri" β•‘ identification (magic number) of the file - 1 Byte β•‘ Version β•‘ the file version - β• Allocator════════╣ - 8 Byte β•‘ NextFreeAddr β•‘ the next free address - 8 Byte β•‘ FreeTermAddr β•‘ the address to the list with free term node segments - 8 Byte β•‘ FreePostingAddr β•‘ the address to the list with free posting node segments - 8 Byte β•‘ FreePositionAddrβ•‘ the address to the list with free position segments - β• Statistic════════╣ - 4 Byte β•‘ Count β•‘ the number of terms in the file - β• Body═════════════╣ - β•‘ Term β•‘ the root node - β•š~~~~~~~~~~~~~~~~~╝ -``` - -The term structure contains the root node, which in turn manages the term nodes. - -``` - β•”Term═══════════╗ - 30 Byte β•‘ TermNode β•‘ the root node - β• Data═══════════╣ - β•‘ β•‘ a variable memory area in which the data is stored - β•š~~~~~~~~~~~~~~~╝ -``` - -The tree structure enables efficient search and retrieval of terms. Each node in the tree represents a character of the term, and the -sequence of characters along the path from the root node to a specific node forms the corresponding term. The `TermNode` segments in -the data area of the reverse index is organized in such a way that they enable a fast and accurate search. A term segment contains -important metadata. This includes the frequency of the term’s occurrence and a reference to a linked list. This list contains the -documents in which the term appears. - -``` - β•”TermNode═══════╗ - 2 Byte β•‘ Character β•‘ a character from the term - 8 Byte β•‘ SiblingAddr β•‘ address of the first sibling node or 0 if no sibling node exists - 8 Byte β•‘ ChildAddr β•‘ address of the first child node or 0 if not present - 4 Byte β•‘ Fequency β•‘ the number of times the term is used - 8 Byte β•‘ PostingAddr β•‘ adress of the first posting node of a binary tree or 0 if there is no element - β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• -``` - -The posting node segment is designed as a binary tree and contains the ids of the documents that belong to a term. For each document, the -posting node segment refers to the position information that indicates where the term is located in the document. The posting segment -is stored in the variable memory area of the inverted index. - -``` - β•”PostingNode════╗ - 16 Byte β•‘ Id β•‘ guid of the document item - 8 Byte β•‘ LeftAddr β•‘ pointer to the address of the left child or 0 if there is no element - 8 Byte β•‘ RightAddr β•‘ pointer to the address of the right child or 0 if there is no element - 8 Byte β•‘ PositionAddr β•‘ adress of the first position element of a sorted list or 0 if there is no element - β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• -``` - -The position segments form a linked list containing the position information of the associated terms. The position of a term -refers to its original occurrence in the field value of a document. Each position segment has a fixed size and is created in -the variable data area of the reverse index. This structure allows for efficient searching and retrieval of terms based on their -position in the documents. - -``` - β•”Position═══════╗ - 4 Byte β•‘ Position β•‘ the position - 8 Byte β•‘ SuccessorAddr β•‘ pointer to the address of the next element of a sorted list or 0 if there is no element - β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• -``` - -**Add**: The procedure for adding terms from an 'IndexField' and saving references to the document with its position within the document -is as follows: - -``` - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ start β”‚ - β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ loop over terms β”‚ retrieve all terms from the new IndexField - β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ tn := gettermnode(term) β”‚ - β”‚ β”‚ β”‚ if tn != null β”‚ - β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ β”‚ p := getposting(id) β”‚ - β”‚ β”‚ β”‚ β”‚ if p != null β”‚ - β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ β”‚ β”‚ add position β”‚ add position associated with the existing posting - β”‚ β”‚ β”‚ β”‚ └─────────────────────── - β”‚ β”‚ β”‚ β”‚ else β”‚ - β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ β”‚ β”‚ add posting β”‚ add posting with the document id - β”‚ β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ add position β”‚ add position associated with the new posting - β”‚ β”‚ β”‚ β”‚ └─┴───────────────────── - β”‚ β”‚ β”‚ β”‚ end if β”‚ - β”‚ β”‚ β”‚ └───────────────────────── - β”‚ β”‚ β”‚ else β”‚ - β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ β”‚ add term node β”‚ add new term node - β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ β”‚ β”‚ add posting β”‚ add posting with the document id - β”‚ β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ add position β”‚ add position associated with the new posting - β”‚ β”‚ β”‚ └─┴─┴───────────────────── - β”‚ β”‚ β”‚ end if β”‚ - β”‚ β”‚ └─────────────────────────── - β”‚ β”‚ end loop β”‚ - β”‚ └───────────────────────────── - β”‚ end β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -**Update**: Updating an `IndexField` in a document is done by determining the difference between the saved and changed terms. All -postings (including positions) will be deleted if they no longer exist in the changed `IndexField`. At the same time, -new postings (including positions) are created for terms that were not included in the original `IndexField`. - -``` - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ current β”‚ - β”‚ β”‚ - β”‚ to delete = β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ current\changed β”‚ β”‚ changed β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ - β”‚ to add = β”‚ - β”‚ changed\current β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` -This results in the following process: -``` - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ start β”‚ - β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ deleteTerms := current\changed β”‚ - β”‚ β”‚ addTerms := changed\current β”‚ - β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ loop over deleteTerms β”‚ - β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ β”‚ delete posting β”‚ remove all postings with the document id - β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ β”‚ β”‚ delete position β”‚ remove all positions associated with the deleted postings - β”‚ β”‚ β”‚ └─┴─────────────────────────── - β”‚ β”‚ β”‚ end loop β”‚ - β”‚ β”‚ └─────────────────────────────── - β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ loop over addTerms β”‚ - β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ β”‚ add posting β”‚ add posting with the document id - β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ β”‚ β”‚ add position β”‚ add position associated with the posting - β”‚ β”‚ β”‚ └─┴─────────────────────────── - β”‚ β”‚ β”‚ end loop β”‚ - β”‚ └─┴─────────────────────────────── - β”‚ end β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -**Remove**: The removal of an IndexField from a reverse index is carried out according to the following procedure: - -``` - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ start β”‚ - β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ loop over terms β”‚ retrieve all terms from the stored IndexField - β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ loop over termnode(term) β”‚ retrieve all TermNodes that correspond to the term - β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ β”‚ delete posting β”‚ remove all postings with the document id - β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ - β”‚ β”‚ β”‚ β”‚ β”‚ delete position β”‚ remove all positions associated with the deleted postings - β”‚ β”‚ β”‚ └─┴─────────────────────── - β”‚ β”‚ β”‚ end loop β”‚ - β”‚ β”‚ └─────────────────────────── - β”‚ β”‚ end loop β”‚ - β”‚ └───────────────────────────── - β”‚ end β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -## Indexing -Indexing is a crucial process that enables quick information retrieval. The index is created from the values of the document -fields. This index is stored on the file system and is updated whenever a document value is added or changed. Sometimes it -is necessary to manually regenerate the index, for example, when a new document field is added or when the index is lost or -damaged. The reindexing deletes all indexes and recreates them. - -```csharp -public class Greetings : IIndexItem - [IndexIgnore] - public Guid Id { get; set; } - - public string Text { get; set; } -} - -// somewhere in the code... -IndexManager.Register(CultureInfo.GetCultureInfo("en"), IndexType.Storage); - -var greetings = new [] -{ - new Greetings { Id = new Guid("b2e8a5c3-1f6d-4e7b-9e1f-8c1a9d0f2b4a"), Text = "Hello Helena!"}, - new Greetings { Id = new Guid("c7d8f9e0-3a2b-4c5d-8e6f-9a1b0c2d4e5f"), Text = "Hello Helena and Helge!"} -}; - -IndexManager.ReIndex(greetings); -``` -``` - β”ŒTerm: 23┐ - β”‚ null β”‚ root - β”‚ 0 β”‚ - β”‚ 53 β”‚β”€β”€β”€β”€β–Ίβ”ŒTerm: 53┐ - β”‚ 0 β”‚ β”‚ 'h' β”‚ first letter from helge and helena - β”‚ 0 β”‚ β”‚ 0 β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 83 β”‚β”€β”€β”€β”€β–Ίβ”ŒTerm: 83┐ - β”‚ 0 β”‚ β”‚ 'e' β”‚ second letter from helge and helena - β”‚ 0 β”‚ β”‚ 0 β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 113 β”‚β”€β”€β”€β”€β–Ίβ”ŒTerm:113┐ - β”‚ 0 β”‚ β”‚ 'l' β”‚ third letter from helge and helena - β”‚ 0 β”‚ β”‚ 0 β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 321 β”‚β”€β”€β”€β”€β–Ίβ”ŒTerm:143┐ - β”‚ 0 β”‚ β”‚ 'e' β”‚ fourth letter from helena - β”‚ 0 β”‚ β”Œβ”€β”€β”‚ 321 β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ 173 β”‚β”€β”€β”€β”€β–Ίβ”ŒTerm:173┐ - β”‚ β”‚ 0 β”‚ β”‚ 'n' β”‚ fifth letter from helena - β”‚ β”‚ 0 β”‚ β”‚ 0 β”‚ - β”ŒTerm:321β”β—„β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 203 β”‚β”€β”€β”€β”€β–Ίβ”ŒTerm:203┐ - β”‚ 'g' β”‚ fourth letter from helge β”‚ 0 β”‚ β”‚ 'a' β”‚ sixth letter from helena - β”‚ 0 β”‚ β”‚ 0 β”‚ β”‚ 0 β”‚ - β”‚ 351 β”‚β”€β”€β”€β”€β–Ίβ”ŒTerm:351┐ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 0 β”‚ - β”‚ 0 β”‚ β”‚ 'e' β”‚ fifth letter from helge β”‚ 2 β”‚ - β”‚ 0 β”‚ β”‚ 0 β”‚ β”Œβ”€β”‚ 233 β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 0 β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ 1 β”‚ β”‚ - β”Œβ”€β”‚ 381 β”‚ β”‚ - β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β–Ό - β–Ό β”ŒPost:233┐ - β”ŒPost:381┐ β”‚ 'b2..' β”‚ - β”‚ 'c7..' β”‚ β”‚ 0 β”‚ - β”‚ 0 β”‚ β”‚ 277 β”‚β”€β”€β”€β”€β–Ίβ”ŒPost:277┐ - β”‚ 0 β”‚ β”ŒPos: 265┐◄────│ 265 β”‚ β”‚ 'c7..' β”‚ - β”ŒPos: 413┐◄────│ 413 β”‚ β”‚ 1 β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 0 β”‚ - β”‚ 3 β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 0 β”‚ β”‚ 0 β”‚ - β”‚ 0 β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 309 β”‚β”€β”€β”€β”€β–Ίβ”ŒPos: 309┐ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 1 β”‚ - β”‚ 0 β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -# WQL -The WebExpress Query Language (WQL) is a query language that filters and sorts of a given amount of data from the reverse index. A statement -of the query language is usually sent from the client to the server, which collects, filters and sorts the data in the reverse index and -sends it back to the client. The following BNF is used to illustrate the grammar: - -``` - ::= | Ξ΅ - ::= "(" ")" | | | Ξ΅ - ::= | "(" ")" - ::= "and" | "or" | "&" | "||" - ::= - ::= | | """ """ | "'" "'" | - ::= | | | | Ξ΅ - ::= "~" - ::= ":" - ::= "(" ")" | Name "(" ")" - ::= "," | Ξ΅ - ::= "=" | ">" | "<" | ">=" | "<=" | "!=" | "~" | "is" | "is not" - ::= "in" | "not in" - ::= "order" "by" | Ξ΅ - ::= "," | Ξ΅ - ::= "asc" | "desc" | Ξ΅ - ::= | | Ξ΅ - ::= "take" | "skip" - ::= [A-Za-z_.][A-Za-z0-9_.]+ - ::= [A-Za-z0-9_@<>=~$%/!+.,;:\-]+ - ::= [+-]?[0-9]*[.]?[0-9]+ - ::= [0-9]+ -``` - -## Term modifiers -Term modifiers in WQL are special characters or combinations of characters that serve to modify search terms, thus offering a wide -range of search possibilities. The use of term modifiers can contribute to improving the efficiency and accuracy of the search. They -can be used to find exact matches for phrases, to search for terms that match a certain pattern, to search for terms that are similar -to a certain value, and to search for terms that are near another term. Term modifiers are an essential part of WQL and contribute to -increasing the power and flexibility of the search. They allow users to create customized search queries tailored to their specific -requirements. It is important to note that all queries are case-insensitive. This means that the case is not considered in the -search, which simplifies the search and improves user-friendliness. - -**Phrase search (exact word sequence)** - -Phrase search allows users to retrieve content from documents that contain a specific order and combination of words defined by the -user. With phrase search, only records that contain the expression in exactly the searched order are returned. For this, the position -information of the reverse index is used. - -`Description = 'lorem ipsum'` - -**Proximity search** - -A proximity search looks for documents where two or more separately matching terms occur within a certain distance of each other. The -distance is determined by the number of intervening words. Proximity search goes beyond simple word matching by adding the constraint -of proximity. By limiting proximity, search results can be avoided where the words are scattered and do not cohere. The basic linguistic -assumption of proximity search is that the proximity of words in a document implies a relationship between the words. - -`Description ~ 'lorem ipsum' :2` - -**Wildcard search** - -A wildcard search is an advanced search technique used to maximize search results. Wildcards are used in search terms to represent -one or more other characters. - -- An asterisk `*` can be used to specify any number of characters. -- A question mark `?` can be used to represent a single character anywhere in the word. It is most useful when there is variable - spellings for a word and you want to search all variants at once. - -`Description ~ '?orem'` -`Description ~ 'ips*'` - -**Fuzzy search** -Fuzzy search is used to find matches in texts that are not exact, but only approximate. - -`Description ~ 'house' ~80` - -**Word search** - -Word search is the search for specific terms in a document, regardless of their capitalization or position. This concept is particularly -useful when searching for specific terms in a document without having to pay attention to their exact spelling or occurrence in the -document. It enables efficient searches for specific terms. - -`Description ~ 'lorem ipsum'` diff --git a/docs/api/toc.yml b/docs/api/toc.yml new file mode 100644 index 0000000..20c32af --- /dev/null +++ b/docs/api/toc.yml @@ -0,0 +1,2 @@ +### YamlMime:TableOfContent +[] diff --git a/docs/assets/webexpress.ico b/docs/assets/webexpress.ico new file mode 100644 index 0000000..c0909c9 Binary files /dev/null and b/docs/assets/webexpress.ico differ diff --git a/docs/assets/webexpress.svg b/docs/assets/webexpress.svg new file mode 100644 index 0000000..e9243f5 --- /dev/null +++ b/docs/assets/webexpress.svg @@ -0,0 +1,40 @@ + + + + + + + + + + template + + + Tabelle.2 + + + + Tabelle.35 + + + + diff --git a/docs/concept.md b/docs/concept.md new file mode 100644 index 0000000..f12d832 --- /dev/null +++ b/docs/concept.md @@ -0,0 +1,1142 @@ +![WebExpress](https://raw.githubusercontent.com/ReneSchwarzer/WebExpress.Doc/main/assets/banner.png) + +# WebExpress.WebIndex +The index model provides a reverse index to enable fast and efficient searching. A reverse +index can significantly speed up access to the data. However, creating and storing a reverse +index requires additional storage space and processing time. The storage requirement increases, +especially with large amounts of data, which can be important. Therefore, it is important to +weigh the pros and cons to achieve the best possible performance. The reverse index offers the +added value of a fast and resource-saving full-text search and accepts the costs mentioned. The +full-text search in `WebExpress` supports the following search options: + +- Word search +- Wildcard search +- Phrase search (exact word sequence) +- Proximity search +- Fuzzy search + +The indexing process begins with the analysis of documents, where the documents are broken +down into smaller units, usually words or phrases. These broken-down units are then converted +into normalized tokens. Normalization can take various forms, such as converting all letters +to lowercase, removing punctuation, or reducing words to their stem. In addition, stop words +are removed. Stop words are frequently occurring words like "and", "is", "in", which typically +do not provide added value for the search. These words are filtered out to improve efficiency +and reduce storage requirements. In many languages, words can appear in different forms that +all refer to the same concept. Therefore, techniques such as stemming, or lemmatization are +often applied to reduce different forms of a word to a common representation. These additional +steps help improve the accuracy and relevance of search results and help keep the index compact +and manageable. The normalized tokens are stored in a reverse index. This index is structured +so that it lists the documents in which each token appears for each token. In addition to the +tokens, more information is stored, such as the frequency of the tokens or the position of +the tokens in the document. During the search process, the search words are tokenized and +normalized in the same way, and then each token is looked up in the reverse index. The documents +or positions found in the lists of all tokens are the search results. This method allows for a +fast and efficient search, as the time-consuming part is carried out in advance during the +indexing process and the actual search consists only of quick lookup operations in the reverse +index. + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” indexing +β”‚ document β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β” searching β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” ╔══════════╗ +β”‚ query β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€>β”‚ tokenizer β”œβ”€β”€β”€β”€β”€β”€>β”‚ stemming, lemmatization & stoppword filter pipe β”œβ”€β”€β”€β”€β”€β”€>β•‘ WebIndex β•‘ +β””β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β•šβ•β•β•β•β•β•β•β•β•β•β• + β–² results β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +Stemming and lemmatization are text preprocessing techniques in natural language processing (NLP). They +reduce the inflected forms of words in a text dataset to a common root form or dictionary form, also +known as a "lemma" in computational linguistics. + +Stemming usually refers to a crude heuristic process that trims the ends of words in the hope +of achieving this goal mostly correctly, and often involves removing derivational affixes. Stemming +algorithms attempt to find the common roots of different inflections by cutting off the endings or +beginnings of the word. + +Lemmatization usually refers to doing things correctly, using a vocabulary and morphological analysis +of words, usually with the aim of removing only inflectional endings and returning the base or dictionary +form of a word, known as a lemma. Unlike stemming, which operates on a single word without knowledge of +the context, lemmatization can distinguish between words that have different meanings depending on the +part of speech. + +These techniques are particularly useful in information search systems such as search engines, where +users can submit a query using one word (e.g., "meditate") but expect results that use any inflected +form of the word (e.g. "meditated", "meditation", etc). + +In this instance, indexing is performed on two documents by executing a series of operations: tokenization, +normalization, and stop-word removal. The outcome of these operations is a multi-dimensional table, which +serves as a representation of the reverse index. + +``` +β”Œdocument a────────────────────────────────────────┐ β”Œdocument b────────────────────────────────────────┐ +β”‚ No, fine, no , good, fine, good. You know Marty, β”‚ β”‚ Thanks a lot, kid. Now, of course not, Biff, now,β”‚ +β”‚ you look so familiar, do I know your mother? Hey β”‚ β”‚ I wouldn't want that to happen. I'm gonna ram β”‚ +β”‚ man, the dance is over. Unless you know someone β”‚ β”‚ him. Well, Marty, I want to thank you for all β”‚ +β”‚ else who could play the guitar. Who's are these? β”‚ β”‚ your good advice, I'll never forget it. Doc, β”‚ +β”‚ Maybe you were adopted. β”‚ β”‚ look, all we need is a little plutonium. β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ + β”‚ β”‚ + β–Ό β–Ό +β”Œnormalized document a─────────────────────────────┐ β”Œnormalized document b─────────────────────────────┐ +β”‚ no fine no good fine good you know marty β”‚ β”‚ thank a lot kid now of course not biff now β”‚ +β”‚ you look so familiar do i know your mother hey β”‚ β”‚ i would not want that to happen i am gonna ram β”‚ +β”‚ man the dance is over unless you know someone β”‚ β”‚ him well marty i want to thank you for all β”‚ +β”‚ else who could play the guitar who is are these β”‚ β”‚ your good advice i will never forget it doc β”‚ +β”‚ maybe you were adopted β”‚ β”‚ look all we need is a little plutonium β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ + β”‚ β”‚ + β–Ό β–Ό +β”Œstopword cleaned document a───────────────────────┐ β”Œstopword cleaned document b───────────────────────┐ +β”‚ fine good fine good know marty β”‚ β”‚ thank lot kid course biff β”‚ +β”‚ look familiar know mother β”‚ β”‚ would want happen gonna ram β”‚ +β”‚ dance unless know someone β”‚ β”‚ well marty want thank β”‚ +β”‚ else could play guitar these β”‚ β”‚ good advice never forget doc β”‚ +β”‚ maybe adopted β”‚ β”‚ look need little plutonium β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ + β”‚ β”‚ + └───────────────┐ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β–Ό β–Ό + β•”WebIndex═══════════════════════════════════════╗ + β•‘ Term β”‚ Fequency β”‚ Documnet β”‚ Position β•‘ + ║═══════════════════════════════════════════════║ + β•‘ adopted β”‚ 1 β”‚ a β”‚ 211 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ advice β”‚ 1 β”‚ b β”‚ 153 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ biff β”‚ 1 β”‚ b β”‚ 40 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ could β”‚ 1 β”‚ a β”‚ 156 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ course β”‚ 1 β”‚ b β”‚ 28 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ dance β”‚ 1 β”‚ a β”‚ 108 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ doc β”‚ 1 β”‚ b β”‚ 183 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ else β”‚ 1 β”‚ a β”‚ 147 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ familiar β”‚ 1 β”‚ a β”‚ 62 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ fine β”‚ 2 β”‚ a β”‚ 5 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ forget β”‚ 1 β”‚ b β”‚ 22 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ gonna β”‚ 1 β”‚ b β”‚ 87 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ good β”‚ 3 β”‚ a β”‚ 16, 28 β•‘ + β•‘ β”‚ β”‚ b β”‚ 148 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ guitar β”‚ 1 β”‚ a β”‚ 171 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ happen β”‚ 1 β”‚ b β”‚ 75 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ kid β”‚ 1 β”‚ b β”‚ 15 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ know β”‚ 3 β”‚ a β”‚ 38, 77, 134 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ little β”‚ 1 β”‚ b β”‚ 211 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ look β”‚ 2 β”‚ a β”‚ 54 β•‘ + β•‘ β”‚ β”‚ b β”‚ 188 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ lot β”‚ 1 β”‚ b β”‚ 10 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ marty β”‚ 2 β”‚ a β”‚ 43 β•‘ + β•‘ β”‚ β”‚ b β”‚ 108 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ maybe β”‚ 1 β”‚ a β”‚ 196 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ mother β”‚ 1 β”‚ a β”‚ 87 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ need β”‚ 1 β”‚ b β”‚ 201 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ never β”‚ 1 β”‚ b β”‚ 166 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ play β”‚ 1 β”‚ a β”‚ 162 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ plutonium β”‚ 1 β”‚ b β”‚ 218 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ ram β”‚ 1 β”‚ b β”‚ 93 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ someone β”‚ 1 β”‚ a β”‚ 139 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ thank β”‚ 2 β”‚ b β”‚ 0, 125 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ these β”‚ 1 β”‚ a β”‚ 189 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ unless β”‚ 1 β”‚ a β”‚ 123 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ want β”‚ 2 β”‚ b β”‚ 62, 117 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ well β”‚ 1 β”‚ b β”‚ 102 β•‘ + ║───────────┼──────────┼──────────┼─────────────║ + β•‘ would β”‚ 1 β”‚ b β”‚ 53 β•‘ + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• + β–² + β”‚ + β”‚ + β”Œstop word cleaned query──┐ + β”‚ marty play guitar β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β–² + β”‚ + β”‚ + β”Œnormalized query─────────┐ + β”‚ marty play the guitar β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β–² + β”‚ + β”‚ + β”Œquery───────┴─────────────┐ + β”‚ 'Marty play the guitar.' β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +`WebIndex` is an efficient system that combines document store and reverse indices to support +a variety of search options. The `IndexDocumentStore` stores all instances of a document for +quick access, regardless of other persistent storage forms such as databases. On the other hand, +the reverse index is created for each field `IndexField` of a document unless it is marked with +`IndexIgnore`. The field contents are tokenized, normalized, and filtered to create the terms of +the reverse index. Each term in the reverse index is associated with a posting that contains the +IDs of the document instances that contain the term. The position where the term was found within +the attribute value is stored in the position. There can be multiple positions for each posting. +When searching for one or more terms, the IDs of the instances and their positions within the +attribute values can be determined. + +``` +β•”IndexManager══════════════════════════════════════════╗ +β•‘ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β•‘ +β•‘ β”‚ WebIndex β”‚ β•‘ +β•‘ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β•‘ +β•‘ 1 β”‚ β•‘ +β•‘ β”‚ β”ŒIndexDocumentStore---------┐ β•‘ +β•‘ * β–Ό Β¦ Β¦ β•‘ +β•‘ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” 1 Β¦ * β”Œβ”€β”€β”€β”€β”€β”€β” Β¦ β•‘ +β•‘ β”‚ IndexDocument β”œβ”€β”€β”€β”€β”€β”€β–Ίβ”‚ Item β”‚ Β¦ β•‘ +β•‘ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ Β¦ β””β”€β”€β”€β”€β”€β”€β”˜ Β¦ β•‘ +β•‘ 1 β”‚ β””---------------------------β”˜ β•‘ +β•‘ β”‚ β•‘ +β•‘ * β–Ό β•‘ +β•‘ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β•‘ +β•‘ β”‚ IndexField β”‚ β•‘ +β•‘ β””β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”˜ β•‘ +β•‘ 1 β”‚ β•‘ +β•‘ β”Œ------β”‚--------IndexReverse┐ β•‘ +β•‘ Β¦ * β–Ό Β¦ β•‘ +β•‘ Β¦ β”Œβ”€β”€β”€β”€β”€β”€β” Β¦ β•‘ +β•‘ Β¦ β”‚ Term β”‚ Β¦ β•‘ +β•‘ Β¦ β””β”€β”€β”€β”¬β”€β”€β”˜ Β¦ β•‘ +β•‘ Β¦ 1 β”‚ Β¦ β•‘ +β•‘ Β¦ β”‚ Β¦ β•‘ +β•‘ Β¦ * β–Ό Β¦ β•‘ +β•‘ Β¦ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” Β¦ β•‘ +β•‘ Β¦ β”‚ Posting β”‚ Β¦ β•‘ +β•‘ Β¦ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”˜ Β¦ β•‘ +β•‘ Β¦ 1 β”‚ Β¦ β•‘ +β•‘ Β¦ β”‚ Β¦ β•‘ +β•‘ Β¦ * β–Ό Β¦ β•‘ +β•‘ Β¦ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” Β¦ β•‘ +β•‘ Β¦ β”‚ Position β”‚ Β¦ β•‘ +β•‘ Β¦ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ Β¦ β•‘ +β•‘ β””---------------------------β”˜ β•‘ +β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• +``` + +# IndexManager +The index manager is a central component of the `WebIndex` system and serves as the primary interface +for interacting with the indexing functions. It is responsible for managing the various `IndexDocuments` +that are created in `WebIndex`. Each `IndexDocument` represents a collection of documents that need to +be indexed, and the index manager ensures that these documents are indexed correctly and efficiently. In +addition, the index manager provides functions for adding, updating, and deleting documents in the +index. It also enables the execution of search queries on the index and returns the corresponding +results. Finally, the index manager provides high control over the indexing process by allowing certain +fields to be excluded from indexing or determining whether the index should be created in main memory or +persistently in the file system. An `IndexDocument` created in main memory enables faster indexing and +searching. However, the number of objects it can support is limited and depends on the size of the +available main memory. Therefore, it is important to weigh the pros and cons and choose the best solution +for the specific requirements. The following diagram serves as a guide to estimate the performance and +resources required when using the file-based approach: + +``` + TIME CHART >01:50 + [h] β–² [ms] β–“ + β”‚ β–“ + 0:24 β”Ό 12 β‰ˆ + 0:23 ─ β–“ + 0:22 β”Ό 11 β–“ + 0:21 ─ β–“ + 0:20 β”Ό 10 β–“ + 0:19 ─ β–“ + 0:18 β”Ό 9 β–“ + 0:17 ─ β–“ + 0:16 β”Ό 8 β–“ + 0:15 ─ β–“ + 0:14 β”Ό 7 β–“ + 0:13 ─ β–“ + 0:12 β”Ό 6 0:11 β–“ + 0:11 ─ 0:10 β–“ β–“>5 + 0:10 β”Ό 5 β–“ β–“ β–“β–‘ + 0:09 ─ 0:08 β–“ β–“ β–“β–‘ + 0:08 β”Ό 4 0:07 β–“ β–“ β–“ β–“β–‘ + 0:06 ─ 0:06 β–“ β–“ β–“ β–“ β–“β–‘ + 0:06 β”Ό 3 0:05 β–“ β–“ β–“ β–“ β–“ β–“β–‘ + 0:05 ─ 0:04 β–“ β–“ β–“ β–“4 β–“ β–“ β–“β–‘ + 0:04 β”Ό 2 0:03 β–“ β–“ β–“ β–“ β–“β–‘ β–“3 β–“ β–“β–‘ + 0:03 ─ 2 0:02 β–“ β–“ β–“2 β–“2 β–“2 β–“β–‘ β–“β–‘ β–“2 β–“β–‘ + 0:02 β”Ό 1 0:01β–‘ β–“1 β–“1 β–“1 β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ + 0:01 ─ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ β–“β–‘ + └──────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬───~~~───┬─────► + 10,000 20,000 30,000 40,000 50,000 60,000 70,000 80,000 90,000 100,000 1,000,000 [items] + + β–“ - time required to reindex in [h] + β–‘ - time required to retrieve in [ms] +``` + +``` + STORAGE SPACE CHART >5,000 + [MB] β–² β–’ + β”‚ 573 β‰ˆ + 550 ─ β–’ β–’ + β”‚ 516 β–’ β–’ + 500 ─ β–’ β–’ 1,525β–’ + β”‚ 460 β–’ β–’ β–ˆβ–’ + 450 ─ β–’ β–’ β–’ β‰ˆβ–’ + β”‚ β–’ β–’ β–’ β–ˆβ–’ + 400 ─ 403 β–’ β–’ β–’ β–ˆβ–’ + β”‚ β–’ β–’ β–’ β–’ β–ˆβ–’ + 350 ─ 345 β–’ β–’ β–’ β–’ β–ˆβ–’ + β”‚ β–’ β–’ β–’ β–’ β–’ β–ˆβ–’ + 300 ─ 291 β–’ β–’ β–’ β–’ β–’ β–ˆβ–’ + β”‚ β–’ β–’ β–’ β–’ β–’ β–’ β–ˆβ–’ + 250 ─ β–’ β–’ β–’ β–’ β–’ β–’ β–ˆβ–’ + β”‚ 234 β–’ β–’ β–’ β–’ β–’ β–’ β–ˆβ–’ + 200 ─ β–’ β–’ β–’ β–’ β–’ β–’ β–’ β–ˆβ–’ + β”‚ 178 β–’ β–’ β–’ β–’ β–’ β–’ β–’ β–ˆβ–’ + 150 ─ β–’ β–’ β–’ β–’ β–’ β–’ β–’ 152β–’ β–ˆβ–’ + β”‚ 121 β–’ β–’ β–’ β–’ β–’ 122β–’ 137β–’ β–ˆβ–’ β–ˆβ–’ + 100 ─ β–’ β–’ β–’ β–’ β–’ 107β–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ + β”‚ 65 β–’ β–’ 61β–’ 76β–’ 91β–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ + 50 ─ 15β–’ 30β–’ 45β–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ + β”‚ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ β–ˆβ–’ + └──────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────┬───~~~───┬─────► + 10,000 20,000 30,000 40,000 50,000 60,000 70,000 80,000 90,000 100,000 1,000,000 [items] + + β–ˆ - payload in [MB], calculation: payload = item count * (100 wrods * 15 character + 99 spaces) / 1024 / 1024 + β–’ - storage space required in [MB] +``` + +It should be noted that the actual execution times and memory consumption depend heavily on the platform +on which WebExpress-WebIndex is operated. In addition, the vocabulary, the content of the documents and +the number of IndexFields have a significant impact on the performance of the system. The measured values +apply to the following conditions: The document contains a field with 100 words each consisting of 15 +characters. The words come from a vocabulary of 20,000 words. The series of measurements was created on +the following system: + +- OS: Windows 11 64bit +- System: Intel NUC-13 +- Processor: Intel i7-1360P +- RAM: 64,0 GB (SODIMM 3200 MHz) RAM +- HDD: Samsung SSD 990 Pro 2TB + +The `IndexManager` class offers a variety of functions for managing and optimizing indexed data. Here are some of the main +methods and properties of this class: + +- `IIndexContext Context`: The context of the index. +- `void Initialization(IIndexContext context)`: Initialization of the IndexManager. +- `void RegisterPipeState(IIndexPipeStage pipeStage)`: Registers a pipe state for processing the tokens. +- `void RegisterWqlFunction()`: Registers a wql function. +- `void ReIndex(IEnumerable items)`: Reindexing of the index. +- `Task ReIndexAsync(IEnumerable items, IProgress progress, CancellationToken token)`: Performs an asynchronous reindexing of a collection of index items. +- `void Create(CultureInfo culture, IndexType type)`: Registers a data type in the index. +- `void Close()`: Closes the index file of type T. +- `Task CloseAsync()`: Asynchronously closes the index file of type T. +- `void Drop()`: Drops all index documents of type T. +- `Task DropAsync()`: Asynchronously drops all index documents of type T. +- `void Insert(TIndexItem item)`: Adds an item to the index. +- `Task InsertAsync(TIndexItem item)`: Performs an asynchronous addition of an item in the index. +- `void Update(TIndexItem item)`: Updates an item in the index. +- `Task UpdateAsync(TIndexItem item)`: Performs an asynchronous update of an item in the index. +- `void Delete(TIndexItem item)`: Removes an item from the index. +- `uint Count()`: Counts the number of items of the index. +- `Task CountAsync()`: Performs an asynchronous determination of the number of elements. +- `Task DeleteAsync(TIndexItem item)`: Removes an item from the index asynchronously. +- `void Clear()`: Removed all data from the index. +- `Task ClearAsync()`: Removed all data from the index asynchronously. +- `IWqlStatement Retrieve(string wql)`: Executes a WQL statement. +- `Task> RetrieveAsync(string wql)`: Executes a wql statement asynchronously. +- `IEnumerable All()`: Returns all documents from the index. +- `IIndexDocument GetIndexDocument()`: Returns an index type based on its type. +- `void Dispose()`: Disposes of the resources used by the current instance. + +## IndexDocument +An `IndexDocument` representing a class that implements the `IIndexItem` interface. Each `IndexDocument` +contains a collection of fields that hold the data to be indexed. These fields can contain various types +of data, such as text, numbers, or dates. During the indexing process, the data in these fields are +analyzed and tokenized, then stored in the reverse index. In addition to the reverse index, an `IndexDocument` +also includes a document store for quick access to the existing instances. When a search query is made for +one or more terms, the IDs of the instances that match the terms are identified in the reverse index and +supplemented with the corresponding instances in the document store and returned to the searcher. + +## IndexField +An `IndexField` is a property (C# property) in an index document that can accommodate various types +of values, such as text, numbers, or other data. Each field is stored in a reverse index, which converts +the field values into terms and associates them with document IDs and positional data in the reverse +index. The name and type of the field are essential pieces of information used during indexing and searching. +If a field is marked with the IndexIgnore attribute, it will be excluded from the indexing process. + +``` + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ <> β”‚ + β”‚ IndexDocument β”œβ”€β”€β”€β”€β”€β”€β”€β”€ IndexDocumentStore β”‚ β”‚ IIndexItem β”‚ + β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β–² + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” Β¦ + β”‚ Property 1 β”‚ Property 2 β”‚ Property … β”‚ Property n β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”Œβ”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β” β”‚ MyIndexItem β”‚ + β”‚ IndexField 1 β”‚ β”‚ IndexField 2 β”‚ β”‚ IndexField … β”‚ β”‚ IndexField n β”‚ <--> β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ + β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ Property 1 β”‚ + β”‚ β”‚ β”‚ β”‚ β”‚ Property 2 β”‚ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ Property … β”‚ +β”‚ IndexReverse 1 β”‚ β”‚ IndexReverse 2 β”‚ β”‚ IndexReverse … β”‚ β”‚ IndexReverse n β”‚ β”‚ Property n β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` +## IndexSchema +The index schema file contains important metadata, which provides detailed information about the +structure and characteristics of the provide indexes. In addition, the file contains a precise object +description of the document captured by the index and is managed. Furthermore, the JSON (JavaScript +Object Notation) format is used for the index schema file and the have the extension `*.ws`. + +## IndexStore +In a filesystem where the `WebIndex` is stored, a process is carried out where an inverted index +is created for each field. These indexes are stored as files with the `.wri` +extension. In parallel, a special storage area known as the document store `.wds` is set +up for each document. In this storage area, the document’s data is redundantly stored to enable quick +access. The structure of these files follows a uniform format that is divided into various segments. Each +of these segments is identifiable by a unique address and has a specific length. There is the option to +specify whether a particular segment should be stored in the cache. If a segment is no longer needed, it +can be removed from the main memory. These features contribute to efficient use of storage and improve +the system performance. + +``` + β•”Header═════════════╗ + 3 Byte β•‘ "wds"|"wrt"|"wrn" β•‘ identification (magic number) of the file `wds` for IndexDocumentStore and `wrt` or 'wrn' for IndexReverse + 1 Byte β•‘ Version β•‘ the file version + β• Allocator══════════╣ + 8 Byte β•‘ NextFreeAddr β•‘ the next free address + 8 Byte β•‘ FreeListAddr β•‘ the address to the free list + β• Statistic══════════╣ + 4 Byte β•‘ Count β•‘ the number of elements in the file + β• Body═══════════════╣ + β•‘ β•‘ a variable memory area in which the data is stored + β•š~~~~~~~~~~~~~~~~~~~╝ +``` + +Unused memory areas in the file are represented by the `Free` segment, which is located in the body area +variable and forms a linked list. The `Allocator` points to the first element of this list. + +``` + β•”Free═══════════════╗ + 8 Byte β•‘ SuccessorAddr β•‘ pointer to the address of the next element of a sorted list or 0 if there is no element + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• +``` + +When new documents are indexed, the new segments are accommodated in a free storage area in the file. Initially, +this is the end of the file. In fragmented files, where segments have already been deleted, the freed storage +areas are reused. The `Allocator` in the header always points to the next free storage space with `NextFreeAddr`. + +### Example alloc +In this example, segments 2 and 3 are successively added. It is important to note that segment 1 already exists. + +``` + initial add segment 2 add segment 3 + + 0 β•”Header═════════════╗ β•”Header═════════════╗ β•”Header═════════════╗ + β•‘ "wds"|"wrt"|"wrn" β•‘ β•‘ "wds"|"wrt"|"wrn" β•‘ β•‘ "wds"|"wrt"|"wrn" β•‘ + 3 β• Allocator══════════╣ β• Allocator══════════╣ β• Allocator══════════╣ + β•‘ 2 β•‘ β•‘ 3 β•‘ β•‘ 4 β•‘ + β•‘ 0 β•‘ β•‘ 0 β•‘ β•‘ 0 β•‘ +19 β• Statistic══════════╣ β• Statistic══════════╣ β• Statistic══════════╣ + β•‘ 1 β•‘ β•‘ 2 β•‘ β•‘ 3 β•‘ +23 β• Body═══════════════╣ β• Body═══════════════╣ β• Body═══════════════╣ + β•‘ β”ŒSeg: 1┐ β•‘ => β•‘ β”ŒSeg: 1┐ β•‘ => β•‘ β”ŒSeg: 1┐ β•‘ + β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ + β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• β•‘ β”ŒSeg: 2┐ β•‘ β•‘ β”ŒSeg: 2┐ β•‘ + β•‘ β”‚ + β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ + β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• β•‘ β”ŒSeg: 3┐ β•‘ + β•‘ β”‚ + β”‚ β•‘ + β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• +``` + +Free spaces are stored in a linked list that represents the free segments in the file. These can be reused +to store new data. Unused Segments are replaced by the `free` segment. + +### Example Free +In this example, segments 2, 1 and 4 are sequentially released and consolidated. + +``` + initial remove segment 2 remove segment 1 remove segment 4 + + 0 β•”Header═════════════╗ β•”Header═════════════╗ β•”Header═════════════╗ β•”Header═════════════╗ + β•‘ "wds"|"wrt"|"wrn" β•‘ β•‘ "wds"|"wrt"|"wrn" β•‘ β•‘ "wds"|"wrt"|"wrn" β•‘ β•‘ "wds"|"wrt"|"wrn" β•‘ + 3 β• Allocator══════════╣ β• Allocator══════════╣ β• Allocator══════════╣ β• Allocator══════════╣ + β•‘ 5 β•‘ β•‘ 5 β•‘ β•‘ 5 β•‘ β•‘ 5 β•‘ + β•‘ 0 β•‘ β•‘ 2 ║─┐ β•‘ 1 ║─┐ β•‘ 4 ║───┐ +19 β• Statistic══════════╣ β• Statistic══════════╣ β”‚ β• Statistic══════════╣ β”‚ β• Statistic══════════╣ β”‚ + β•‘ 4 β•‘ β•‘ 3 β•‘ β”‚ β•‘ 2 β•‘ β”‚ β•‘ 1 β•‘ β”‚ +23 β• Body═══════════════╣ β• Body═══════════════╣ β”‚ β• Body═══════════════╣ β”‚ β• Body═══════════════╣ β”‚ + β•‘ β”ŒSeg: 1┐ β•‘ => β•‘ β”ŒSeg: 1┐ β•‘ β”‚ => β•‘ β”ŒFree: 1┐ β•‘β—„β”˜ => β•‘ β”ŒFree: 1┐ ║◄─┐│ + β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β”‚ β•‘ β”‚ X β”‚ ║─┐ β•‘ β”‚ β”‚ ║─┐││ + β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚β”‚β”‚ + β•‘ β”ŒSeg: 2┐ β•‘ β•‘ β”ŒFree: 2┐ β•‘β—„β”˜ β•‘ β”ŒFree: 2┐ β•‘β—„β”˜ β•‘ β”ŒFree: 2┐ β•‘β—„β”˜β”‚β”‚ + β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ X β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β”‚β”‚ + β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚β”‚ + β•‘ β”ŒSeg: 3┐ β•‘ β•‘ β”ŒSeg: 3┐ β•‘ β•‘ β”ŒSeg: 3┐ β•‘ β•‘ β”ŒSeg: 3┐ β•‘ β”‚β”‚ + β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β”‚β”‚ + β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚β”‚ + β•‘ β”ŒSeg: 4┐ β•‘ β•‘ β”ŒSeg: 4┐ β•‘ β•‘ β”ŒSeg: 4┐ β•‘ β•‘ β”ŒFree: 4┐ β•‘β—„β”€β”Όβ”˜ + β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ β”‚ β•‘ β•‘ β”‚ X β”‚ β•‘β”€β”€β”˜ + β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• +``` + +The repurposing of unused segments reduces the space requirements of files, particularly for highly fluctuating +index files. This practice not only optimizes storage but also enhances the overall performance and efficiency +of data management systems. It is especially beneficial in environments where data is frequently updated or +deleted, leading to a high turnover of index files. By reusing these segments, the system can maintain optimal +performance while minimizing the need for additional storage space. + +### Example realloc +This example reallocates a segment using the available free space. Since the size of the segments is fixed, +it is irrelevant which free one Segment is reused. For efficiency reasons, the first element from the list +of free segments is always used. + +``` + initial realloc segment 4 + + 0 β•”Header═════════════╗ β•”Header═════════════╗ + β•‘ "wds"|"wrt"|"wrn" β•‘ β•‘ "wds"|"wrt"|"wrn" β•‘ + 3 β• Allocator══════════╣ β• Allocator══════════╣ + β•‘ 5 β•‘ β•‘ 5 β•‘ + β•‘ 4 ║───┐ β•‘ 1 ║─┐ +19 β• Statistic══════════╣ β”‚ β• Statistic══════════╣ β”‚ + β•‘ 1 β•‘ β”‚ β•‘ 2 β•‘ β”‚ +23 β• Body═══════════════╣ β”‚ β• Body═══════════════╣ β”‚ + β•‘ β”ŒFree: 1┐ ║◄─┐│=> β•‘ β”ŒSeg: 1─┐ β•‘β—„β”˜ + β•‘ β”‚ β”‚ ║─┐││ β•‘ β”‚ β”‚ ║─┐ + β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚β”‚β”‚ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚ + β•‘ β”ŒFree: 2┐ β•‘β—„β”˜β”‚β”‚ β•‘ β”ŒFree: 2┐ β•‘β—„β”˜ + β•‘ β”‚ β”‚ β•‘ β”‚β”‚ β•‘ β”‚ β”‚ β•‘ + β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚β”‚ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ + β•‘ β”ŒSeg: 3┐ β•‘ β”‚β”‚ β•‘ β”ŒSeg: 3┐ β•‘ + β•‘ β”‚ β”‚ β•‘ β”‚β”‚ β•‘ β”‚ β”‚ β•‘ + β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β”‚β”‚ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ + β•‘ β”ŒFree: 4┐ β•‘β—„β”€β”Όβ”˜ β•‘ β”ŒSeg: 4┐ β•‘ + β•‘ β”‚ β”‚ β•‘β”€β”€β”˜ β•‘ β”‚ x β”‚ β•‘ + β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ β•‘ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β•‘ + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• +``` + +### Caching +Caching is an efficient technique for optimizing data access by enabling fast access to frequently used +data and simultaneously reducing the load on the file system. It stores frequently used data in memory, +which speeds up access to this data as it does not have to be retrieved from the hard drive again. For +write accesses, the data is first written to the read cache. The read cache uses a hash map to allow +random access to the cached segments. Each cached segment has a defined lifetime. If this has expired, +the segments are removed from the read cache, unless they have been marked as immortal via the `SegmentCached` +attribute. The maximum size of the read cache can be determined using the `IndexStorageReadBuffer.MaxCachedSegments` +parameter. The size influences all `IndexStore` files and can be changed during operation. However, it should +be noted that if the size of memory already allocated will not be released. + + +## IndexDocumentStore +A `IndexDocumentStore` is a data structure in which each key is associated with a value. This allows +efficient retrieval and retrieval of data based on the key. The document store plays a crucial role in +improving the efficiency of queries by enabling direct access to the document instances that contain the +desired terms. The internal structure of the document store: + +``` + β•”Header═════════════╗ + 3 Byte β•‘ "wds" β•‘ identification (magic number) of the file + 1 Byte β•‘ Version β•‘ the file version + β• Allocator══════════╣ + 8 Byte β•‘ NextFreeAddr β•‘ the next free address + 8 Byte β•‘ FreeItemAddr β•‘ the address to the list with free item node segments + 8 Byte β•‘ FreeChunkAddr β•‘ the address to the list with free chunk node segments + β• Statistic══════════╣ + 4 Byte β•‘ Count β•‘ the number of terms in the file + β• Body═══════════════╣ + β•‘ HashMap β•‘ a hash map in which the data is stored + β•š~~~~~~~~~~~~~~~~~~~╝ +``` + +Access to the document instances is done via a HashMap, where the document id serves as the key. + +``` + β•”HashMap════════════╗ + 4 Byte β•‘ BucketCount β•‘ number of buckets (the next prime number of capacity) + β• Buckets════════════╣ + β•‘ Bucket 0 β•‘ slot in which items with the same hash value are stored + n * β•‘ Bucket 1 β•‘ + 8 Byte β•‘~~~~~~~~~~~~~~~~~~~β•‘ + β•‘~~~~~~~~~~~~~~~~~~~β•‘ + β•‘ Bucket n-1 β•‘ + β• Data═══════════════╣ + β•‘ β•‘ a variable memory area in which the data is stored + β•š~~~~~~~~~~~~~~~~~~~╝ +``` + +Each bucket contains a pointer to blocks of a list, with the elements of the bucket. + +``` + β•”Bucket═════════════╗ + 8 Byte β•‘ ItemAddr β•‘ pointer to the address of the first element of a sorted list that has the same hash values (collisions) or 0 if there is no element + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• +``` + +The document instances are stored in one or more segments. The size of the segment is fixed and the number of +segments per record is determined by the size of the compressed document instance. The segment is stored in +the variable storage area. + +``` + β•”Item═══════════════╗ + 16 Byte β•‘ Id β•‘ guid of the document item + 4 Byte β•‘ Length β•‘ size of the DataChunk in bytes +256 Byte β•‘ DataChunk β•‘ a memory area in which a part of the element is stored (gzip compressed) + 8 Byte β•‘ NextChunkAddr β•‘ pointer to the address of the next chunk element of a list or 0 if there is no element + 8 Byte β•‘ SuccessorAddr β•‘ pointer to the address of the next bucket element of a sorted list or 0 if there is no element + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• +``` + +If the data is larger than what can fit in a chunk, additional chunks are created and the data is divided among +them. The last chunk may not be completely filled. All chunks are linked in an ordered list. + +``` + β•”Chunk══════════════╗ + 4 Byte β•‘ Length β•‘ size of the DataChunk in bytes +256 Byte β•‘ DataChunk β•‘ a memory area in which a part of the element is stored (gzip compressed) + 8 Byte β•‘ NextChunkAddr β•‘ pointer to the address of the next chunk element of a list or 0 if there is no element + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• +``` + +**Add**: The add function in the `IndexDocumentStore` is designed to permanently store the entire document. To +efficiently utilize storage space and minimize storage requirements, the object is serialized into a JSON format +and subsequently compressed. This process ensures efficient use of storage space without compromising the integrity +or accessibility of the data. Furthermore, this method allows for faster data transmission and enhances the overall +performance of the `IndexDocumentStore`. Access to the original data can be obtained at any time through +decompression and deserialization. + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ start β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ if !contains(id) β”‚ look up document id in hash map +β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ gzip(data) β”‚ gzip the data +β”‚ β”‚ β”‚ add item β”‚ adding an items segment +β”‚ β”‚ └─────────────────────────── +β”‚ β”‚ else β”‚ +β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ throw ArgumentException β”‚ +β”‚ β”‚ └─────────────────────────── +β”‚ β”‚ end if β”‚ +β”‚ └───────────────────────────── +β”‚ end β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +**Update**: The update process consists of a combination of delete and add operations. If the data is of +the same size, the existing item segment is reused. Otherwise, a new item segment is created and used. This +approach optimizes storage usage and enhances system efficiency by avoiding unnecessary storage allocations. + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ start β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ if contains(id) β”‚ look up document id in hash map +β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ delete β”‚ delete item +β”‚ β”‚ β”‚ gzip(data) β”‚ gzip the data +β”‚ β”‚ β”‚ delete item β”‚ remove the existing item segment +β”‚ β”‚ β”‚ add item β”‚ adding the updated item segment +β”‚ β”‚ └─────────────────────────── +β”‚ β”‚ else β”‚ +β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ add β”‚ add item +β”‚ β”‚ └─────────────────────────── +β”‚ β”‚ end if β”‚ +β”‚ └───────────────────────────── +β”‚ end β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +**Remove**: Documents that are no longer needed can be securely removed from the document storage by using +the delete function. This ensures efficient use of storage and keeps the document storage tidy and +well-organized. + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ start β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ if !contains(id) β”‚ look up document id in hash map +β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ delete item β”‚ remove the existing item segment +β”‚ β”‚ └─────────────────────────── +β”‚ β”‚ else β”‚ +β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ throw ArgumentException β”‚ +β”‚ β”‚ └─────────────────────────── +β”‚ β”‚ end if β”‚ +β”‚ └───────────────────────────── +β”‚ end β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### IndexReverse term +A reverse index is a specialized type of index that allows access to data in reverse order. In the context of +the `WebIndex`, the reverse index is used for efficient searching of terms. These terms are derived from the +associated fields, and their values are broken down into tokens, normalized, filtered, and stored in a search +tree for fast retrieval. + +Unlike the general definition of the header, `IndexReverse` has a modified allocator. A separate list is kept +for each segment type in which the free segments are saved. For faster storage and reuse, insertion and removal +only ever occurs at the beginning of the list. When there is a new memory request, a free segment can be reused +without any search effort. Merging segments is not necessary. + +``` + β•”Header═════════════╗ + 3 Byte β•‘ "wrt" β•‘ identification (magic number) of the file + 1 Byte β•‘ Version β•‘ the file version + β• Allocator══════════╣ + 8 Byte β•‘ NextFreeAddr β•‘ the next free address + 8 Byte β•‘ FreeTermAddr β•‘ the address to the list with free term node segments + 8 Byte β•‘ FreePostingAddr β•‘ the address to the list with free posting node segments + 8 Byte β•‘ FreePositionAddr β•‘ the address to the list with free position segments + β• Statistic══════════╣ + 4 Byte β•‘ Count β•‘ the number of terms in the file + β• Body═══════════════╣ + β•‘ Term β•‘ the root node + β•š~~~~~~~~~~~~~~~~~~~╝ +``` + +The term structure contains the root node, which in turn manages the term nodes. + +``` + β•”Term═══════════════╗ + 30 Byte β•‘ TermNode β•‘ the root node + β• Data═══════════════╣ + β•‘ β•‘ a variable memory area in which the data is stored + β•š~~~~~~~~~~~~~~~~~~~╝ +``` + +The tree structure enables efficient search and retrieval of terms. Each node in the tree represents a character +of the term, and the sequence of characters along the path from the root node to a specific node forms the corresponding +term. The `TermNode` segments in the data area of the reverse index is organized in such a way that they enable a fast +and accurate search. A term segment contains important metadata. This includes the frequency of the term’s occurrence and +a reference to a linked list. This list contains the documents in which the term appears. + +``` + β•”TermNode═══════════╗ + 2 Byte β•‘ Character β•‘ a character from the term + 8 Byte β•‘ SiblingAddr β•‘ address of the first sibling node or 0 if no sibling node exists + 8 Byte β•‘ ChildAddr β•‘ address of the first child node or 0 if not present + 4 Byte β•‘ Fequency β•‘ the number of times the term is used + 8 Byte β•‘ PostingAddr β•‘ adress of the first posting node of a binary tree or 0 if there is no element exists + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• +``` + +The posting node segment is designed as a binary tree and contains the ids of the documents that belong to a term. For +each document, the posting node segment refers to the position information that indicates where the term is located in +the document. The posting segment is stored in the variable memory area of the inverted index. + +``` + β•”TermPostingNode════╗ + 16 Byte β•‘ Id β•‘ guid of the document item + 8 Byte β•‘ LeftAddr β•‘ pointer to the address of the left child or 0 if there is no element exists + 8 Byte β•‘ RightAddr β•‘ pointer to the address of the right child or 0 if there is no element exists + 8 Byte β•‘ PositionAddr β•‘ adress of the first position element of a sorted list or 0 if there is no element exists + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• +``` + +The position segments form a linked list containing the position information of the associated terms. The position of a term +refers to its original occurrence in the field value of a document. Each position segment has a fixed size and is created in +the variable data area of the reverse index. This structure allows for efficient searching and retrieval of terms based on their +position in the documents. + +``` + β•”Position═══════════╗ + 4 Byte β•‘ Position β•‘ the position + 8 Byte β•‘ SuccessorAddr β•‘ pointer to the address of the next element of a sorted list or 0 if there is no element exists + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• +``` + +**Add**: The procedure for adding terms from an 'IndexField' and saving references to the document with its position within +the document is as follows: + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ start β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ loop over terms β”‚ retrieve all terms from the new IndexField +β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ tn := gettermnode(term) β”‚ +β”‚ β”‚ β”‚ if tn != null β”‚ +β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ β”‚ p := getposting(id) β”‚ +β”‚ β”‚ β”‚ β”‚ if p != null β”‚ +β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ β”‚ β”‚ add position β”‚ add position associated with the existing posting +β”‚ β”‚ β”‚ β”‚ └─────────────────────── +β”‚ β”‚ β”‚ β”‚ else β”‚ +β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ β”‚ β”‚ add posting β”‚ add posting with the document id +β”‚ β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ add position β”‚ add position associated with the new posting +β”‚ β”‚ β”‚ β”‚ └─┴───────────────────── +β”‚ β”‚ β”‚ β”‚ end if β”‚ +β”‚ β”‚ β”‚ └───────────────────────── +β”‚ β”‚ β”‚ else β”‚ +β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ β”‚ add term node β”‚ add new term node +β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ β”‚ β”‚ add posting β”‚ add posting with the document id +β”‚ β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ add position β”‚ add position associated with the new posting +β”‚ β”‚ β”‚ └─┴─┴───────────────────── +β”‚ β”‚ β”‚ end if β”‚ +β”‚ β”‚ └─────────────────────────── +β”‚ β”‚ end loop β”‚ +β”‚ └───────────────────────────── +β”‚ end β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +**Update**: Updating an `IndexField` in a document is done by determining the difference between the saved and changed +terms. All postings (including positions) will be deleted if they no longer exist in the changed `IndexField`. At the +same time, new postings (including positions) are created for terms that were not included in the original `IndexField`. + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ current β”‚ +β”‚ β”‚ +β”‚ to delete = β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ current\changed β”‚ β”‚ changed β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ + β”‚ to add = β”‚ + β”‚ changed\current β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +This results in the following process: + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ start β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ deleteTerms := current\changed β”‚ +β”‚ β”‚ addTerms := changed\current β”‚ +β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ loop over deleteTerms β”‚ +β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ β”‚ delete posting β”‚ remove all postings with the document id +β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ β”‚ β”‚ delete position β”‚ remove all positions associated with the deleted postings +β”‚ β”‚ β”‚ └─┴─────────────────────────── +β”‚ β”‚ β”‚ end loop β”‚ +β”‚ β”‚ └─────────────────────────────── +β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ loop over addTerms β”‚ +β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ β”‚ add posting β”‚ add posting with the document id +β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ β”‚ β”‚ add position β”‚ add position associated with the posting +β”‚ β”‚ β”‚ └─┴─────────────────────────── +β”‚ β”‚ β”‚ end loop β”‚ +β”‚ └─┴─────────────────────────────── +β”‚ end β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +**Remove**: The removal of an IndexField from a reverse index is carried out according to the following procedure: + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ start β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ loop over terms β”‚ retrieve all terms from the stored IndexField +β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ loop over termnode(term) β”‚ retrieve all TermNodes that correspond to the term +β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ β”‚ delete posting β”‚ remove all postings with the document id +β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ β”‚ β”‚ β”‚ β”‚ delete position β”‚ remove all positions associated with the deleted postings +β”‚ β”‚ β”‚ └─┴─────────────────────── +β”‚ β”‚ β”‚ end loop β”‚ +β”‚ β”‚ └─────────────────────────── +β”‚ β”‚ end loop β”‚ +β”‚ └───────────────────────────── +β”‚ end β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### IndexReverse numeric +Unlike reverse indices based on terms, the reverse index based on numerical values offers significant advantages when +indexing numerical data. The management structures for numerical reverse indices differ substantially: Instead of a term +tree, a balanced binary tree is used, in which each node contains the complete numerical value. This structure enables +more efficient management and faster search for numerical values. Balanced binary trees ensure that the tree height remains +logarithmic, allowing for quick search, insert, and delete operations. This method offers clear performance advantages in +processing and managing numerical data compared to term-based trees. + +``` + β•”Header═════════════╗ + 3 Byte β•‘ "wrn" β•‘ identification (magic number) of the file + 1 Byte β•‘ Version β•‘ the file version + β• Allocator══════════╣ + 8 Byte β•‘ NextFreeAddr β•‘ the next free address + 8 Byte β•‘ FreeTermAddr β•‘ the address to the list with free term node segments + 8 Byte β•‘ FreePostingAddr β•‘ the address to the list with free posting node segments + 8 Byte β•‘ FreePositionAddr β•‘ the address to the list with free position segments + β• Statistic══════════╣ + 4 Byte β•‘ Count β•‘ the number of terms in the file + β• Body═══════════════╣ + β•‘ Numeric β•‘ the root node + β•š~~~~~~~~~~~~~~~~~~~╝ +``` + +The numeric structure contains the root node, which in turn manages the numeric nodes. + +``` + β•”Numeric════════════╗ + 44 Byte β•‘ NumericNode β•‘ the root node + β• Data═══════════════╣ + β•‘ β•‘ a variable memory area in which the data is stored + β•š~~~~~~~~~~~~~~~~~~~╝ +``` + +The tree structure enables efficient search and retrieval of numeric values by using a balanced binary tree to store the +values. Each node has a pointer to a posting tree where the document ids of the associated documents are stored. + +``` + β•”NumericNode════════╗ + 16 Byte β•‘ Value β•‘ the numeric value of the node + 8 Byte β•‘ LeftAddr β•‘ address of the left child node or 0 if no left child node exists + 8 Byte β•‘ RightdAddr β•‘ address of the right child node or 0 if not right child node exists + 4 Byte β•‘ Fequency β•‘ the number of times the numeric value is used + 8 Byte β•‘ NumPostingNode β•‘ Address of the first posting node of a binary tree or 0 if no element exists + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• +``` + +The posting node segment is designed as a binary tree and contains the ids of the documents that belong to a term. For +each document, the posting node segment refers to the position information that indicates where the term is located in +the document. The posting segment is stored in the variable memory area of the inverted index. + +``` + β•”NumericPostingNode═╗ + 16 Byte β•‘ Id β•‘ guid of the document item + 8 Byte β•‘ LeftAddr β•‘ pointer to the address of the left child or 0 if there is no element + 8 Byte β•‘ RightAddr β•‘ pointer to the address of the right child or 0 if there is no element + β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β• +``` + +## Indexing +Indexing is a crucial process that enables quick information retrieval. The index is created from the values of +the document fields. This index is stored on the file system and is updated whenever a document value is added or +changed. Sometimes it is necessary to manually regenerate the index, for example, when a new document field is added +or when the index is lost or damaged. The reindexing deletes all indexes and recreates them. + +```csharp +public class Greetings : IIndexItem + [IndexIgnore] + public Guid Id { get; set; } + + public string Text { get; set; } +} + +// somewhere in the code... +IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + +var greetings = new [] +{ + new Greetings { Id = new Guid("b2e8a5c3-1f6d-4e7b-9e1f-8c1a9d0f2b4a"), Text = "Hello Helena!"}, + new Greetings { Id = new Guid("c7d8f9e0-3a2b-4c5d-8e6f-9a1b0c2d4e5f"), Text = "Hello Helena and Helge!"} +}; + +IndexManager.ReIndex(greetings); +``` + +From the data of the example, the following term tree results: + +``` +β”ŒTerm: 23┐ +β”‚ null β”‚ root +β”‚ 0 β”‚ +β”‚ 53 β”‚β”€β”€β”€β”€β–Ίβ”ŒTerm: 53┐ +β”‚ 0 β”‚ β”‚ 'h' β”‚ first letter from helge and helena +β”‚ 0 β”‚ β”‚ 0 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 83 β”‚β”€β”€β”€β”€β–Ίβ”ŒTerm: 83┐ + β”‚ 0 β”‚ β”‚ 'e' β”‚ second letter from helge and helena + β”‚ 0 β”‚ β”‚ 0 β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 113 β”‚β”€β”€β”€β”€β–Ίβ”ŒTerm:113┐ + β”‚ 0 β”‚ β”‚ 'l' β”‚ third letter from helge and helena + β”‚ 0 β”‚ β”‚ 0 β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 321 β”‚β”€β”€β”€β”€β–Ίβ”ŒTerm:143┐ + β”‚ 0 β”‚ β”‚ 'e' β”‚ fourth letter from helena + β”‚ 0 β”‚ β”Œβ”€β”€β”‚ 321 β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ 173 β”‚β”€β”€β”€β”€β–Ίβ”ŒTerm:173┐ + β”‚ β”‚ 0 β”‚ β”‚ 'n' β”‚ fifth letter from helena + β”‚ β”‚ 0 β”‚ β”‚ 0 β”‚ + β”ŒTerm:321β”β—„β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 203 β”‚β”€β”€β”€β”€β–Ίβ”ŒTerm:203┐ + β”‚ 'g' β”‚ fourth letter from helge β”‚ 0 β”‚ β”‚ 'a' β”‚ sixth letter from helena + β”‚ 0 β”‚ β”‚ 0 β”‚ β”‚ 0 β”‚ + β”‚ 351 β”‚β”€β”€β”€β”€β–Ίβ”ŒTerm:351┐ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 0 β”‚ + β”‚ 0 β”‚ β”‚ 'e' β”‚ fifth letter from helge β”‚ 2 β”‚ + β”‚ 0 β”‚ β”‚ 0 β”‚ β”Œβ”€β”‚ 233 β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 0 β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ 1 β”‚ β”‚ + β”Œβ”€β”‚ 381 β”‚ β”‚ + β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β–Ό + β–Ό β”ŒPost:233┐ + β”ŒPost:381┐ β”‚ 'b2..' β”‚ + β”‚ 'c7..' β”‚ β”‚ 0 β”‚ + β”‚ 0 β”‚ β”‚ 277 β”‚β”€β”€β”€β”€β–Ίβ”ŒPost:277┐ + β”‚ 0 β”‚ β”ŒPos: 265┐◄────│ 265 β”‚ β”‚ 'c7..' β”‚ + β”ŒPos: 413┐◄────│ 413 β”‚ β”‚ 1 β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 0 β”‚ + β”‚ 3 β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 0 β”‚ β”‚ 0 β”‚ + β”‚ 0 β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 309 β”‚β”€β”€β”€β”€β–Ίβ”ŒPos: 309┐ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ 1 β”‚ + β”‚ 0 β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +# WQL +The WebExpress Query Language (WQL) is a query language that filters and sorts of a given +amount of data from the reverse index. A statement of the query language is usually sent +from the client to the server, which collects, filters and sorts the data in the reverse +index and sends it back to the client. The following BNF is used to illustrate the grammar: + +``` + ::= | Ξ΅ + ::= "(" ")" | | | Ξ΅ + ::= | "(" ")" + ::= "and" | "or" | "&" | "||" + ::= | "." + ::= | | """ """ | "'" "'" | + ::= | | | | Ξ΅ + ::= "~" + ::= ":" + ::= "(" ")" | Name "(" ")" + ::= "," | Ξ΅ + ::= "=" | ">" | "<" | ">=" | "<=" | "!=" | "~" | "is" | "is not" + ::= "in" | "not in" + ::= "order" "by" | Ξ΅ + ::= "," | Ξ΅ + ::= "asc" | "desc" | Ξ΅ + ::= | | Ξ΅ + ::= "take" | "skip" + ::= [A-Za-z_][A-Za-z0-9_]+ + ::= [A-Za-z0-9_@<>=~$%/!+.,;:\-]+ + ::= [+-]?[0-9]*[.]?[0-9]+ + ::= [0-9]+ +``` + +## Term modifiers +Term modifiers in WQL are special characters or combinations of characters that serve to +modify search terms, thus offering a wide range of search possibilities. The use of term +modifiers can contribute to improving the efficiency and accuracy of the search. They can +be used to find exact matches for phrases, to search for terms that match a certain pattern, +to search for terms that are similar to a certain value, and to search for terms that are +near another term. Term modifiers are an essential part of WQL and contribute to increasing +the power and flexibility of the search. They allow users to create customized search queries +tailored to their specific requirements. It is important to note that all queries are case- +insensitive. This means that the case is not considered in the search, which simplifies the +search and improves user-friendliness. + +**Phrase search (exact word sequence)** + +Phrase search allows users to retrieve content from documents that contain a specific order +and combination of words defined by the user. With phrase search, only records that contain +the expression in exactly the searched order are returned. For this, the position information +of the reverse index is used. + +```wql +Description = 'lorem ipsum' +``` + +**Proximity search** + +A proximity search looks for documents where two or more separately matching terms occur within a +certain distance of each other. The distance is determined by the number of intervening words. Proximity +search goes beyond simple word matching by adding the constraint of proximity. By limiting proximity, +search results can be avoided where the words are scattered and do not cohere. The basic linguistic +assumption of proximity search is that the proximity of words in a document implies a relationship +between the words. + +```wql +Description ~ 'lorem ipsum' :2 +``` + +**Wildcard search** + +A wildcard search is an advanced search technique used to maximize search results. Wildcards are used in search terms to represent +one or more other characters. + +- An asterisk `*` can be used to specify any number of characters. +- A question mark `?` can be used to represent a single character anywhere in the word. It is +most useful when there is variable spellings for a word and you want to search all variants +at once. + +```wql +Description ~ '?orem' +Description ~ 'ips*' +``` + +**Fuzzy search** +Fuzzy search is used to find matches in texts that are not exact, but only approximate. + +```wql +Description ~ 'house' ~80 +``` + +**Word search** + +Word search is the search for specific terms in a document, regardless of their capitalization +or position. This concept is particularly useful when searching for specific terms in a document +without having to pay attention to their exact spelling or occurrence in the document. It enables +efficient searches for specific terms. + +```wql +Description ~ 'lorem ipsum' +``` + +## WQL functions +The WebExpress Query Language (WQL) offers a set of functions that allow data to be processed and retrieved +in versatile and specific ways. These functions can be integrated into queries to achieve more precise and +targeted search results. Below are some common functions and their descriptions: + +| Function | Params | Description +|----------|--------|------------------------------------------------ +| day() | n | Returns n days before or after the current day. +| now() | - | Returns the current date and time. + +Functions are only allowed on the right-hand side of conditions. This means that functions always appear as +part of the parameters in conditions and not as standalone left-hand operands. Here are some examples to +illustrate this: + +```wql +dateField = day(-3) +``` + +This query filters all records where the dateField has the value that is 3 days before the current day. + +Custom functions can be created by implementing the `IWqlExpressionNodeFilterFunction` interface and +registering it in the `IndexManager`. + diff --git a/docs/docfx.json b/docs/docfx.json new file mode 100644 index 0000000..fb7f3ba --- /dev/null +++ b/docs/docfx.json @@ -0,0 +1,61 @@ +{ + "$schema": "https://raw.githubusercontent.com/dotnet/docfx/main/schemas/docfx.schema.json", + "metadata": [ + { + "src": [ + { + "files": [ + "src/WebExpress.WebIndex/*.csproj" + ], + "src": "../" + } + ], + "dest": "api", + "outputFormat": "apiPage" + } + ], + "build": { + "content": [ + { + "files": [ "**/*.{md,yml}" ], + "exclude": [ "_site/**", "obj/**" ] + } + ], + "resource": [ + { + "files": [ "**/images/**", "**/media/**", "codesnippet/**" ], + "exclude": [ "_site/**", "obj/**" ] + }, + { + "files": [ "assets/webexpress.ico", "assets/webexpress.svg" ] + }, + { + "src": "../schemas", + "files": [ "**/*.json" ], + "dest": "schemas" + } + ], + "postProcessors": [ "ExtractSearchIndex" ], + "globalMetadata": { + "_appTitle": "WebExpress.WebIndex", + "_appName": "WebExpress.WebIndex", + "_appFaviconPath": "assets/webexpress.ico", + "_appLogoPath": "assets/webexpress.svg", + "pdf": false + }, + "markdownEngineProperties": { + "alerts": { + "TODO": "alert alert-secondary" + } + }, + "xref": [ + "../.xrefmap.json" + ], + "output": "../_site", + "template": [ + "default", + "modern", + "template" + ] + } +} diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..63249d4 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,28 @@ +![WebExpress](https://raw.githubusercontent.com/ReneSchwarzer/WebExpress/main/assets/banner.png) + +# WebExpress +WebExpress is a lightweight web server optimized for use in low-performance environments (e.g. Raspberry PI). By providing +a powerful plugin system and a comprehensive API, web applications can be easily and quickly integrated into a .NET +language (e.g. C#). Some advantages of WebExpress are: + +- It is easy to use. +- It offers a variety of features and tools that can help you build and manage your website. +- It is fast and efficient and can help you save time and money. +- It is flexible and can be customized to meet your specific requirements. + +The `WebExpress` family includes the following projects: + +- [WebExpress](https://github.com/ReneSchwarzer/WebExpress#readme) - The web server for `WebExpress` applications and the documentation. +- [WebExpress.WebCore](https://github.com/ReneSchwarzer/WebExpress.WebCore#readme) - The core for `WebExpress` applications. +- [WebExpress.WebUI](https://github.com/ReneSchwarzer/WebExpress.WebUI#readme) - Common templates and controls for `WebExpress` applications. +- [WebExpress.WebIndex](https://github.com/ReneSchwarzer/WebExpress.WebIndex#readme) - Reverse index for `WebExpress` applications. +- [WebExpress.WebApp](https://github.com/ReneSchwarzer/WebExpress.WebApp#readme) - Business application template for `WebExpress` applications. + +# WebExpress.WebIndex +`WebExpress.WebIndex` is part of the WebExpress family. The project provides a reverse index to enable a quick and efficient search for data. The index can be filtered using the wql (webexpress query language). Even though the reverse index is part of the `WebExpress` family, it can also be used in other projects (outside of `WebExpress`). For detailed information about `WebIndex`, see [concept](https://github.com/ReneSchwarzer/WebExpress.WebIndex/blob/main/docs/concept.md). + +# Download +The current binaries are available for download [here](https://github.com/ReneSchwarzer/WebExpress/releases). + +# Tags +#WebIndex #WebExpress #ReverseIndex #DotNet #NETCore diff --git a/docs/template/dashboard.html.tmpl b/docs/template/dashboard.html.tmpl new file mode 100644 index 0000000..0288fe8 --- /dev/null +++ b/docs/template/dashboard.html.tmpl @@ -0,0 +1,47 @@ +{{!Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license.}} +{{!master(layout/_master.tmpl)}} +

{{title}}

+{{#items.Length}} +
+{{#items}} +
+ {{name}} +
+
{{name}}
+

{{{description}}}

+
+
+ {{#usage}} +
+ {{#config}}
docfx.json: {{config}}
{{/config}} + {{#command}}
docfx: {{command}}
{{/command}} + {{#init}}
docfx init: {{init}}
{{/init}} +
+ {{/usage}} +
+
+{{/items}} +
+{{/items.Length}} + + diff --git a/docs/template/public/main.css b/docs/template/public/main.css new file mode 100644 index 0000000..b0b8c1f --- /dev/null +++ b/docs/template/public/main.css @@ -0,0 +1,10 @@ +/** + * Licensed to the .NET Foundation under one or more agreements. + * The .NET Foundation licenses this file to you under the MIT license. + */ + +/* Checkout https://getbootstrap.com/docs/5.3/customize/color/ for more customization options */ +body { + --bs-link-color-rgb: 66, 184, 131 !important; + --bs-link-hover-color-rgb: 64, 180, 128 !important; +} diff --git a/docs/template/public/main.js b/docs/template/public/main.js new file mode 100644 index 0000000..d5c4867 --- /dev/null +++ b/docs/template/public/main.js @@ -0,0 +1,14 @@ +/** + * Licensed to the .NET Foundation under one or more agreements. + * The .NET Foundation licenses this file to you under the MIT license. + */ + +export default { + iconLinks: [ + { + icon: 'github', + href: 'https://github.com/dotnet/docfx', + title: 'GitHub' + } + ] +} diff --git a/docs/template/schemas/Dashboard.schema.json b/docs/template/schemas/Dashboard.schema.json new file mode 100644 index 0000000..a00ab75 --- /dev/null +++ b/docs/template/schemas/Dashboard.schema.json @@ -0,0 +1,45 @@ +{ + "title": "Dashboard", + "$schema": "https://dotnet.github.io/docfx/schemas/v1.0/schema.json#", + "version": "1.0.0", + "description": "Schema for dashboard", + "id": "https://github.com/dotnet/docfx/schemas/Dashboard.schema.json", + "type": "object", + "properties": { + "uid": { + "type": "string", + "contentType": "uid" + }, + "title": { + "type": "string", + "tags": [ + "localizable" + ] + }, + "description": { + "type": "string", + "contentType": "markdown", + "tags": [ + "localizable" + ] + }, + "items": { + "items": { + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string", + "contentType": "markdown", + "tags": [ + "localizable" + ] + } + }, + "type": "object" + }, + "type": "array" + } + } +} \ No newline at end of file diff --git a/docs/toc.yml b/docs/toc.yml new file mode 100644 index 0000000..1b187bc --- /dev/null +++ b/docs/toc.yml @@ -0,0 +1,10 @@ +ο»Ώ- name: Home + href: index.md +- name: API Documentation + href: api/WebExpress.WebIndex.html +- name: Concept + href: concept.md +- name: User Guide + href: user-guide.md +- name: Tutorials + href: tutorials.md diff --git a/docs/tutorials.md b/docs/tutorials.md new file mode 100644 index 0000000..77c2af0 --- /dev/null +++ b/docs/tutorials.md @@ -0,0 +1,17 @@ +![WebExpress](https://raw.githubusercontent.com/ReneSchwarzer/WebExpress/main/assets/banner.png) + +# Tutorials +Welcome to the `WebExpress` Tutorials! Here, you'll find step-by-step guides and helpful resources to get the most out +of `WebExpress`. Whether you're a beginner just starting out or an experienced developer looking to expand your skills, +our tutorials offer something for everyone. + +# Getting Started +Begin with our basic tutorial: +- [HelloWorld](https://github.com/ReneSchwarzer/WebExpress.Tutorial.HelloWorld#readme) +- [WebApp](https://github.com/ReneSchwarzer/WebExpress.Tutorial.WebApp#readme) +- [WebIndex](https://github.com/ReneSchwarzer/WebExpress.Tutorial.WebIndex#readme) + +This tutorial will guide you through the initial steps of creating and running your first `WebExpress` application. + +Stay tuned for more exciting and educational tutorials to help you unlock the full potential of `WebExpress`. Happy coding and +best of luck with your projects! diff --git a/docs/user-guide.md b/docs/user-guide.md new file mode 100644 index 0000000..99d6a86 --- /dev/null +++ b/docs/user-guide.md @@ -0,0 +1,17 @@ +![WebExpress](https://raw.githubusercontent.com/ReneSchwarzer/WebExpress/main/assets/banner.png) + +# User guide +Welcome to the `WebExpress.WebIndex` User Guide. This guide will help you get started with `WebExpress.WebIndex` and make the most out of its +features. Follow the links below to begin your journey. + +# Getting started +To get started with `WebExpress.WebIndex`, use the following guides: + +- [Installation Guide](https://github.com/ReneSchwarzer/WebExpress/blob/main/doc/installation_guide.md) +- [Development Guide](https://github.com/ReneSchwarzer/WebExpress/blob/main/doc/development_guide.md) +- [WebExpress.WebCore API Documentation](https://reneschwarzer.github.io/WebExpress.WebCore/) +- [WebExpress.WebUI API Documentation](https://reneschwarzer.github.io/WebExpress.WebUI/) +- [WebExpress.WebApp API Documentation](https://reneschwarzer.github.io/WebExpress.WebApp/) +- [WebExpress.WebIndex API Documentation](https://reneschwarzer.github.io/WebExpress.WebIndex/) + +We hope you enjoy using `WebExpress.WebIndex` and find it valuable for your projects. Happy coding! diff --git a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryA.cs b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryA.cs index dc764e5..c4c1cb5 100644 --- a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryA.cs +++ b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryA.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.DocumentStore /// /// Test class for testing the memory-based document store. /// + [Collection("NonParallelTests")] public class UnitTestDocumentStoreMemoryA : UnitTestDocumentStore { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. diff --git a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryB.cs b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryB.cs index 0aeba15..a87ee96 100644 --- a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryB.cs +++ b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryB.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.DocumentStore /// /// Test class for testing the memory-based document store. /// + [Collection("NonParallelTests")] public class UnitTestDocumentStoreMemoryB : UnitTestDocumentStore { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. diff --git a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryC.cs b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryC.cs index 945d6f5..5f88693 100644 --- a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryC.cs +++ b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryC.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.DocumentStore /// /// Test class for testing the memory-based document store. /// + [Collection("NonParallelTests")] public class UnitTestDocumentStoreMemoryC : UnitTestDocumentStore { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. diff --git a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryD.cs b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryD.cs index 9d93fd3..6ca9f4b 100644 --- a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryD.cs +++ b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryD.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.DocumentStore /// /// Test class for testing the memory-based document store. /// + [Collection("NonParallelTests")] public class UnitTestDocumentStoreMemoryD : UnitTestDocumentStore { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. diff --git a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryE.cs b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryE.cs index f276d1a..fe1ae2f 100644 --- a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryE.cs +++ b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryE.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.DocumentStore /// /// Test class for testing the memory-based document store. /// + [Collection("NonParallelTests")] public class UnitTestDocumentStoreMemoryE : UnitTestDocumentStore { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. diff --git a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryF.cs b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryF.cs new file mode 100644 index 0000000..56fc5e1 --- /dev/null +++ b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreMemoryF.cs @@ -0,0 +1,213 @@ +ο»Ώusing WebExpress.WebIndex.Memory; +using WebExpress.WebIndex.Test.Document; +using WebExpress.WebIndex.Test.Fixture; +using Xunit.Abstractions; + +namespace WebExpress.WebIndex.Test.DocumentStore +{ + /// + /// Test class for testing the memory-based document store for unicode. + /// + [Collection("NonParallelTests")] + public class UnitTestDocumentStoreMemoryF : UnitTestDocumentStore + { + /// + /// Initializes a new instance of the class. + /// + /// The log. + /// The test context. + public UnitTestDocumentStoreMemoryF(UnitTestIndexFixtureIndexF fixture, ITestOutputHelper output) + : base(fixture, output) + { + } + + /// + /// Creates a document store. + /// + [Fact] + public void Create() + { + // preconditions + var context = new IndexContext(); + + // test execution + var documentStore = new IndexMemoryDocumentStore(context, (uint)Fixture.TestData.Count); + + // postconditions + documentStore.Dispose(); + } + + /// + /// Adds items to a document store. + /// + [Fact] + public void Add() + { + // preconditions + Preconditions(); + var documentStore = new IndexMemoryDocumentStore(Context, (uint)Fixture.TestData.Count); + + documentStore.Clear(); + + // test execution + foreach (var item in Fixture.TestData) + { + documentStore.Add(item); + } + + var i = documentStore.GetItem(Fixture.TestData[0].Id); + + Assert.True(i != null && i.Id == Fixture.TestData[0].Id); + + // postconditions + documentStore.Dispose(); + Postconditions(); + } + + /// + /// Update an entry in the reverse index where the item has a first name change. + /// + [Fact] + public void UpdateWithChange() + { + // preconditions + Preconditions(); + var documentStore = new IndexMemoryDocumentStore(Context, (uint)Fixture.TestData.Count); + var randomItem = Fixture.RandomItem; + + documentStore.Clear(); + foreach (var item in Fixture.TestData) + { + documentStore.Add(item); + } + + var name = "Update_" + randomItem.Name; + var changed = new UnitTestIndexTestDocumentF + { + Id = randomItem.Id, + Name = name + }; + + // test execution + documentStore.Update(changed); + + var all = documentStore.All; + + Assert.True(all.Select(x => x.Id).OrderBy(x => x).SequenceEqual(Fixture.TestData.Select(x => x.Id).OrderBy(x => x))); + Assert.True(all.Where(x => x.Name == name).Any()); + + // postconditions + documentStore.Dispose(); + Postconditions(); + } + + /// + /// Changes an entry in the reverse index without the element to be changed having any changes. + /// + [Fact] + public void UpdateWithoutChanges() + { + // preconditions + Preconditions(); + var documentStore = new IndexMemoryDocumentStore(Context, (uint)Fixture.TestData.Count); + var randomItem = Fixture.RandomItem; + + documentStore.Clear(); + foreach (var item in Fixture.TestData) + { + documentStore.Add(item); + } + + // test execution + documentStore.Update(randomItem); + var all = documentStore.All; + + Assert.True(all.Select(x => x.Id).OrderBy(x => x).SequenceEqual(Fixture.TestData.Select(x => x.Id).OrderBy(x => x))); + Assert.True(all.Where(x => x.Name == randomItem.Name).Any()); + + // postconditions + documentStore.Dispose(); + Postconditions(); + } + + /// + /// Removes an entry from the document store. + /// + [Fact] + public void Remove() + { + // preconditions + Preconditions(); + var documentStore = new IndexMemoryDocumentStore(Context, (uint)Fixture.TestData.Count); + + documentStore.Clear(); + foreach (var item in Fixture.TestData) + { + documentStore.Add(item); + } + + // test execution + documentStore.Delete(Fixture.TestData[0]); + var all = documentStore.All; + + Assert.True(all.Select(x => x.Id).SequenceEqual(Fixture.TestData.Where(x => x.Id != Fixture.TestData[0].Id).Select(x => x.Id))); + + // postconditions + documentStore.Dispose(); + Postconditions(); + } + + /// + /// Retrieve a entry of the reverse index. + /// + [Fact] + public void Retrieve() + { + // preconditions + Preconditions(); + var documentStore = new IndexMemoryDocumentStore(Context, (uint)Fixture.TestData.Count); + + documentStore.Clear(); + foreach (var document in Fixture.TestData) + { + documentStore.Add(document); + } + + // test execution + var item = documentStore.GetItem(Fixture.TestData[0].Id); + + Assert.NotNull(documentStore); + Assert.NotNull(item); + + // postconditions + documentStore.Dispose(); + Postconditions(); + } + + /// + /// Return all entries of the document store. + /// + [Fact] + public void All() + { + // preconditions + Preconditions(); + var documentStore = new IndexMemoryDocumentStore(Context, (uint)Fixture.TestData.Count); + + documentStore.Clear(); + foreach (var item in Fixture.TestData) + { + documentStore.Add(item); + } + + // test execution + var all = documentStore.All; + + Assert.True(all.Select(x => x.Id).OrderBy(x => x).SequenceEqual(Fixture.TestData.Select(x => x.Id).OrderBy(x => x))); + + // postconditions + documentStore.Dispose(); + Postconditions(); + } + } +} diff --git a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageA.cs b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageA.cs index 4ebbc72..dd2418a 100644 --- a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageA.cs +++ b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageA.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.DocumentStore /// /// Test class for testing the storage-based document store. /// + [Collection("NonParallelTests")] public class UnitTestDocumentStoreStorageA : UnitTestDocumentStore { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. @@ -46,8 +47,6 @@ public void Add() Preconditions(); var documentStore = new IndexStorageDocumentStore(Context, 5); - documentStore.Clear(); - // test execution documentStore.Add(Fixture.TestData[0]); documentStore.Add(Fixture.TestData[1]); @@ -181,7 +180,6 @@ public void All() Preconditions(); var documentStore = new IndexStorageDocumentStore(Context, 5); - documentStore.Clear(); documentStore.Add(Fixture.TestData[0]); documentStore.Add(Fixture.TestData[1]); @@ -194,5 +192,34 @@ public void All() documentStore.Dispose(); Postconditions(); } + + /// + /// Clear the document store. + /// + [Fact] + public void Clear() + { + // preconditions + Preconditions(); + var documentStore = new IndexStorageDocumentStore(Context, 5); + documentStore.Add(Fixture.TestData[2]); + documentStore.Add(Fixture.TestData[3]); + + // test execution + documentStore.Clear(); + Assert.Empty(documentStore.All); + + documentStore.Add(Fixture.TestData[0]); + + documentStore.Add(Fixture.TestData[1]); + + var all = documentStore.All.ToList(); + + Assert.Equal(all.Select(x => x.Id).OrderBy(x => x), Fixture.TestData.Take(2).Select(x => x.Id).OrderBy(x => x)); + + // postconditions + documentStore.Dispose(); + Postconditions(); + } } } diff --git a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageB.cs b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageB.cs index a96a899..de9b89a 100644 --- a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageB.cs +++ b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageB.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.DocumentStore /// /// Test class for testing the storage-based document store. /// + [Collection("NonParallelTests")] public class UnitTestDocumentStoreStorageB : UnitTestDocumentStore { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. diff --git a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageC.cs b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageC.cs index f840faf..a6730a8 100644 --- a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageC.cs +++ b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageC.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.DocumentStore /// /// Test class for testing the storage-based document store. /// + [Collection("NonParallelTests")] public class UnitTestDocumentStoreStorageC : UnitTestDocumentStore { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. diff --git a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageD.cs b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageD.cs index 7255c2d..82ba33e 100644 --- a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageD.cs +++ b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageD.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.DocumentStore /// /// Test class for testing the storage-based document store. /// + [Collection("NonParallelTests")] public class UnitTestDocumentStoreStorageD : UnitTestDocumentStore { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. diff --git a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageE.cs b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageE.cs index 72c1025..3eb86f5 100644 --- a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageE.cs +++ b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageE.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.DocumentStore /// /// Test class for testing the storage-based document store. /// + [Collection("NonParallelTests")] public class UnitTestDocumentStoreStorageE : UnitTestDocumentStore { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. @@ -72,7 +73,7 @@ public void UpdateWithChange() // preconditions Preconditions(); var documentStore = new IndexStorageDocumentStore(Context, (uint)Fixture.TestData.Count); - var randomItem = Fixture.RandomItem; + var randomItem = Fixture.TestData.LastOrDefault(); documentStore.Clear(); foreach (var item in Fixture.TestData) diff --git a/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageF.cs b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageF.cs new file mode 100644 index 0000000..9a71612 --- /dev/null +++ b/src/WebExpress.WebIndex.Test/DocumentStore/UnitTestDocumentStoreStorageF.cs @@ -0,0 +1,242 @@ +ο»Ώusing WebExpress.WebIndex.Storage; +using WebExpress.WebIndex.Test.Document; +using WebExpress.WebIndex.Test.Fixture; +using Xunit.Abstractions; + +namespace WebExpress.WebIndex.Test.DocumentStore +{ + /// + /// Test class for testing the storage-based document store. + /// + [Collection("NonParallelTests")] + public class UnitTestDocumentStoreStorageF : UnitTestDocumentStore + { + /// + /// Initializes a new instance of the class. + /// + /// The log. + /// The test context. + public UnitTestDocumentStoreStorageF(UnitTestIndexFixtureIndexF fixture, ITestOutputHelper output) + : base(fixture, output) + { + } + + /// + /// Creates a document store. + /// + [Fact] + public void Create() + { + // preconditions + Preconditions(); + + // test execution + var documentStore = new IndexStorageDocumentStore(Context, (uint)Fixture.TestData.Count); + + // postconditions + documentStore.Dispose(); + } + + /// + /// Adds items to a document store. + /// + [Fact] + public void Add() + { + // preconditions + Preconditions(); + var documentStore = new IndexStorageDocumentStore(Context, (uint)Fixture.TestData.Count); + + documentStore.Clear(); + + // test execution + foreach (var item in Fixture.TestData) + { + documentStore.Add(item); + } + + var i = documentStore.GetItem(Fixture.TestData[0].Id); + + Assert.True(i != null && i.Id == Fixture.TestData[0].Id); + + // postconditions + documentStore.Dispose(); + Postconditions(); + } + + /// + /// Update an entry in the reverse index where the item has a first name change. + /// + [Fact] + public void UpdateWithChange() + { + // preconditions + Preconditions(); + var documentStore = new IndexStorageDocumentStore(Context, (uint)Fixture.TestData.Count); + var randomItem = Fixture.RandomItem; + + documentStore.Clear(); + foreach (var item in Fixture.TestData) + { + documentStore.Add(item); + } + + var name = "Update_" + randomItem.Name; + var changed = new UnitTestIndexTestDocumentF + { + Id = randomItem.Id, + Name = name + }; + + // test execution + documentStore.Update(changed); + + var all = documentStore.All; + + Assert.Equal(all.Select(x => x.Id).OrderBy(x => x), Fixture.TestData.Select(x => x.Id).OrderBy(x => x)); + Assert.True(all.Where(x => x.Name == name).Any()); + + // postconditions + documentStore.Dispose(); + Postconditions(); + } + + /// + /// Changes an entry in the reverse index without the element to be changed having any changes. + /// + [Fact] + public void UpdateWithoutChanges() + { + // preconditions + Preconditions(); + var documentStore = new IndexStorageDocumentStore(Context, (uint)Fixture.TestData.Count); + var randomItem = Fixture.RandomItem; + + documentStore.Clear(); + foreach (var item in Fixture.TestData) + { + documentStore.Add(item); + } + + // test execution + documentStore.Update(randomItem); + var all = documentStore.All; + + Assert.Equal(all.Select(x => x.Id).OrderBy(x => x), Fixture.TestData.Select(x => x.Id).OrderBy(x => x)); + Assert.True(all.Where(x => x.Name == randomItem.Name).Any()); + + // postconditions + documentStore.Dispose(); + Postconditions(); + } + + /// + /// Removes an entry from the document store. + /// + [Fact] + public void Remove() + { + // preconditions + Preconditions(); + var documentStore = new IndexStorageDocumentStore(Context, (uint)Fixture.TestData.Count); + + documentStore.Clear(); + foreach (var item in Fixture.TestData) + { + documentStore.Add(item); + } + + // test execution + documentStore.Delete(Fixture.TestData[0]); + var all = documentStore.All; + + Assert.Equal(all.Select(x => x.Id).OrderBy(x => x), Fixture.TestData.Where(x => x.Id != Fixture.TestData[0].Id).Select(x => x.Id).OrderBy(x => x)); + + // postconditions + documentStore.Dispose(); + Postconditions(); + } + + /// + /// Retrieve a entry of the reverse index. + /// + [Fact] + public void Retrieve() + { + // preconditions + Preconditions(); + var documentStore = new IndexStorageDocumentStore(Context, (uint)Fixture.TestData.Count); + + documentStore.Clear(); + foreach (var document in Fixture.TestData) + { + documentStore.Add(document); + } + + // test execution + var item = documentStore.GetItem(Fixture.TestData[0].Id); + + Assert.NotNull(documentStore); + Assert.NotNull(item); + + // postconditions + documentStore.Dispose(); + Postconditions(); + } + + /// + /// Return all entries of the document store. + /// + [Fact] + public void All() + { + // preconditions + Preconditions(); + var documentStore = new IndexStorageDocumentStore(Context, (uint)Fixture.TestData.Count); + + documentStore.Clear(); + foreach (var item in Fixture.TestData) + { + documentStore.Add(item); + } + + // test execution + var all = documentStore.All; + + Assert.Equal(all.Select(x => x.Id).OrderBy(x => x), Fixture.TestData.Select(x => x.Id).OrderBy(x => x)); + + // postconditions + documentStore.Dispose(); + Postconditions(); + } + + /// + /// Reopen the document store. + /// + [Fact] + public void ReOpen() + { + // preconditions + Preconditions(); + var documentStore = new IndexStorageDocumentStore(Context, (uint)Fixture.TestData.Count); + foreach (var item in Fixture.TestData) + { + documentStore.Add(item); + } + + documentStore.Dispose(); + + // test execution + documentStore = new IndexStorageDocumentStore(Context, (uint)Fixture.TestData.Count); + + var all = documentStore.All; + + Assert.Equal("wds", documentStore.Header.Identifier); + Assert.Equal(all.Select(x => x.Id).OrderBy(x => x), Fixture.TestData.Select(x => x.Id).OrderBy(x => x)); + + // postconditions + documentStore.Dispose(); + Postconditions(); + } + } +} diff --git a/src/WebExpress.WebIndex.Test/Fixture/NonParallelTestsCollection.cs b/src/WebExpress.WebIndex.Test/Fixture/NonParallelTestsCollection.cs new file mode 100644 index 0000000..270c343 --- /dev/null +++ b/src/WebExpress.WebIndex.Test/Fixture/NonParallelTestsCollection.cs @@ -0,0 +1,11 @@ +ο»Ώnamespace WebExpress.WebIndex.Test.Fixture +{ + /// + /// Defines a collection of tests that should not be run in parallel. + /// + [CollectionDefinition("NonParallelTests", DisableParallelization = true)] + public class NonParallelTestsCollection : ICollectionFixture + { + + } +} diff --git a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixture.cs b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixture.cs index b2cb312..e50445f 100644 --- a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixture.cs +++ b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixture.cs @@ -1,6 +1,9 @@ ο»Ώnamespace WebExpress.WebIndex.Test.Fixture { - public abstract class UnitTestIndexFixture : IDisposable + /// + /// Provides a base class for unit test fixtures that require disposable resources. + /// + public class UnitTestIndexFixture : IDisposable { /// /// The random number generator. @@ -10,6 +13,8 @@ public abstract class UnitTestIndexFixture : IDisposable /// /// Disposes of the resources used by the current instance. /// - public abstract void Dispose(); + public virtual void Dispose() + { + } } } diff --git a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexA.cs b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexA.cs index 76cc793..b15fefd 100644 --- a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexA.cs +++ b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexA.cs @@ -15,7 +15,7 @@ public class UnitTestIndexFixtureIndexA : UnitTestIndexFixture public UnitTestIndexTestDocumentA RandomItem => TestData[Rand.Next(TestData.Count)]; /// - /// Constructor + /// Initializes a new instance of the class. /// public UnitTestIndexFixtureIndexA() { diff --git a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexB.cs b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexB.cs index 346266c..223857b 100644 --- a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexB.cs +++ b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexB.cs @@ -15,7 +15,7 @@ public class UnitTestIndexFixtureIndexB : UnitTestIndexFixture public UnitTestIndexTestDocumentB RandomItem => TestData[Rand.Next(TestData.Count)]; /// - /// Constructor + /// Initializes a new instance of the class. /// public UnitTestIndexFixtureIndexB() { diff --git a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexC.cs b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexC.cs index 8e00ce3..b961d2f 100644 --- a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexC.cs +++ b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexC.cs @@ -16,7 +16,7 @@ public class UnitTestIndexFixtureIndexC : UnitTestIndexFixture public UnitTestIndexTestDocumentC RandomItem => TestData[Rand.Next(TestData.Count)]; /// - /// Constructor + /// Initializes a new instance of the class. /// public UnitTestIndexFixtureIndexC() { diff --git a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexD.cs b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexD.cs index 39d388d..f4cca7a 100644 --- a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexD.cs +++ b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexD.cs @@ -15,7 +15,7 @@ public class UnitTestIndexFixtureIndexD : UnitTestIndexFixture public UnitTestIndexTestDocumentD RandomItem => TestData[Rand.Next(TestData.Count)]; /// - /// Constructor + /// Initializes a new instance of the class. /// public UnitTestIndexFixtureIndexD() { diff --git a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexE.cs b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexE.cs index aa6c7aa..97aadcd 100644 --- a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexE.cs +++ b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexE.cs @@ -15,7 +15,7 @@ public class UnitTestIndexFixtureIndexE : UnitTestIndexFixture public UnitTestIndexTestDocumentE RandomItem => TestData[Rand.Next(TestData.Count)]; /// - /// Constructor + /// Initializes a new instance of the class. /// public UnitTestIndexFixtureIndexE() { diff --git a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexF.cs b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexF.cs new file mode 100644 index 0000000..1cf47d1 --- /dev/null +++ b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureIndexF.cs @@ -0,0 +1,34 @@ +ο»Ώusing WebExpress.WebIndex.Test.Document; + +namespace WebExpress.WebIndex.Test.Fixture +{ + /// + /// Represents a fixture for unit tests related to index functionality. + /// + public class UnitTestIndexFixtureIndexF : UnitTestIndexFixture + { + /// + /// Returns the test data. + /// + public List TestData { get; } = UnitTestIndexTestDocumentFactoryF.GenerateTestData(); + + /// + /// Returns a random document item. + /// + public UnitTestIndexTestDocumentF RandomItem => TestData[Rand.Next(TestData.Count)]; + + /// + /// Initializes a new instance of the class. + /// + public UnitTestIndexFixtureIndexF() + { + } + + /// + /// Disposes of the resources used by the current instance. + /// + public override void Dispose() + { + } + } +} diff --git a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureToken.cs b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureToken.cs index e48f5dd..34be730 100644 --- a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureToken.cs +++ b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureToken.cs @@ -19,7 +19,7 @@ public class UnitTestIndexFixtureToken : UnitTestIndexFixture public IndexTokenAnalyzer TokenAnalyzer { get; private set; } /// - /// Constructor + /// Initializes a new instance of the class. /// public UnitTestIndexFixtureToken() { diff --git a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlA.cs b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlA.cs index 569f306..a9a67da 100644 --- a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlA.cs +++ b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlA.cs @@ -1,4 +1,5 @@ ο»Ώusing System.Globalization; +using System.Reflection; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Wql; @@ -17,13 +18,17 @@ public class UnitTestIndexFixtureWqlA : UnitTestIndexFixture public IEnumerable TestData { get; } = UnitTestIndexTestDocumentFactoryA.GenerateTestData(); /// - /// Constructor + /// Initializes a new instance of the class. /// public UnitTestIndexFixtureWqlA() { var context = new IndexContext(); context.IndexDirectory = Path.Combine(context.IndexDirectory, Path.GetFileNameWithoutExtension(Path.GetRandomFileName())); - IndexManager.Initialization(context); + + // use reflection to call the protected Initialization method + var method = typeof(IndexManagerTest).GetMethod("Initialization", BindingFlags.Instance | BindingFlags.NonPublic); + method.Invoke(IndexManager, [context]); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); IndexManager.ReIndex(TestData); } @@ -40,7 +45,7 @@ public override void Dispose() /// /// Executes a wql statement. /// - /// Tje wql statement. + /// The wql statement. /// The WQL parser. public IWqlStatement ExecuteWql(string wql) { diff --git a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlB.cs b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlB.cs index 154342d..8c2536a 100644 --- a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlB.cs +++ b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlB.cs @@ -1,4 +1,5 @@ ο»Ώusing System.Globalization; +using System.Reflection; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Wql; @@ -17,13 +18,17 @@ public class UnitTestIndexFixtureWqlB : UnitTestIndexFixture public IEnumerable TestData { get; } = UnitTestIndexTestDocumentFactoryB.GenerateTestData(); /// - /// Constructor + /// Initializes a new instance of the class. /// public UnitTestIndexFixtureWqlB() { var context = new IndexContext(); context.IndexDirectory = Path.Combine(context.IndexDirectory, Path.GetFileNameWithoutExtension(Path.GetRandomFileName())); - IndexManager.Initialization(context); + + // use reflection to call the protected Initialization method + var method = typeof(IndexManagerTest).GetMethod("Initialization", BindingFlags.Instance | BindingFlags.NonPublic); + method.Invoke(IndexManager, [context]); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); IndexManager.ReIndex(TestData); } @@ -40,7 +45,7 @@ public override void Dispose() /// /// Executes a wql statement. /// - /// Tje wql statement. + /// The wql statement. /// The WQL parser. public IWqlStatement ExecuteWql(string wql) { diff --git a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlC.cs b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlC.cs index f632788..4f6dfaf 100644 --- a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlC.cs +++ b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlC.cs @@ -1,4 +1,5 @@ ο»Ώusing System.Globalization; +using System.Reflection; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Wql; @@ -27,13 +28,17 @@ public class UnitTestIndexFixtureWqlC : UnitTestIndexFixture public string Term { get; private set; } /// - /// Constructor + /// Initializes a new instance of the class. /// public UnitTestIndexFixtureWqlC() { var context = new IndexContext(); context.IndexDirectory = Path.Combine(context.IndexDirectory, Path.GetFileNameWithoutExtension(Path.GetRandomFileName())); - IndexManager.Initialization(context); + + // use reflection to call the protected Initialization method + var method = typeof(IndexManagerTest).GetMethod("Initialization", BindingFlags.Instance | BindingFlags.NonPublic); + method.Invoke(IndexManager, [context]); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); IndexManager.ReIndex(TestData); RandomItem = TestData.Skip(Rand.Next(TestData.Count())).FirstOrDefault(); @@ -52,7 +57,7 @@ public override void Dispose() /// /// Executes a wql statement. /// - /// Tje wql statement. + /// The wql statement. /// The WQL parser. public IWqlStatement ExecuteWql(string wql) { diff --git a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlD.cs b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlD.cs index 1073f0a..04ac6b6 100644 --- a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlD.cs +++ b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlD.cs @@ -1,4 +1,5 @@ ο»Ώusing System.Globalization; +using System.Reflection; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Wql; @@ -17,13 +18,17 @@ public class UnitTestIndexFixtureWqlD : UnitTestIndexFixture public IEnumerable TestData { get; } = UnitTestIndexTestDocumentFactoryD.GenerateTestData(); /// - /// Constructor + /// Initializes a new instance of the class. /// public UnitTestIndexFixtureWqlD() { var context = new IndexContext(); context.IndexDirectory = Path.Combine(context.IndexDirectory, Path.GetFileNameWithoutExtension(Path.GetRandomFileName())); - IndexManager.Initialization(context); + + // use reflection to call the protected Initialization method + var method = typeof(IndexManagerTest).GetMethod("Initialization", BindingFlags.Instance | BindingFlags.NonPublic); + method.Invoke(IndexManager, [context]); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); IndexManager.ReIndex(TestData); } @@ -40,7 +45,7 @@ public override void Dispose() /// /// Executes a wql statement. /// - /// Tje wql statement. + /// The wql statement. /// The WQL parser. public IWqlStatement ExecuteWql(string wql) { diff --git a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlE.cs b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlE.cs index 3c7ecb6..d96aeee 100644 --- a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlE.cs +++ b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlE.cs @@ -1,4 +1,5 @@ ο»Ώusing System.Globalization; +using System.Reflection; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Wql; @@ -17,13 +18,17 @@ public class UnitTestIndexFixtureWqlE : UnitTestIndexFixture public IEnumerable TestData { get; } = UnitTestIndexTestDocumentFactoryE.GenerateTestData(); /// - /// Constructor + /// Initializes a new instance of the class. /// public UnitTestIndexFixtureWqlE() { var context = new IndexContext(); context.IndexDirectory = Path.Combine(context.IndexDirectory, Path.GetFileNameWithoutExtension(Path.GetRandomFileName())); - IndexManager.Initialization(context); + + // use reflection to call the protected Initialization method + var method = typeof(IndexManagerTest).GetMethod("Initialization", BindingFlags.Instance | BindingFlags.NonPublic); + method.Invoke(IndexManager, [context]); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); IndexManager.ReIndex(TestData); } @@ -40,7 +45,7 @@ public override void Dispose() /// /// Executes a wql statement. /// - /// Tje wql statement. + /// The wql statement. /// The WQL parser. public IWqlStatement ExecuteWql(string wql) { diff --git a/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlF.cs b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlF.cs new file mode 100644 index 0000000..2d5ac30 --- /dev/null +++ b/src/WebExpress.WebIndex.Test/Fixture/UnitTestIndexFixtureWqlF.cs @@ -0,0 +1,58 @@ +ο»Ώusing System.Globalization; +using System.Reflection; +using WebExpress.WebIndex.Test.Document; +using WebExpress.WebIndex.Wql; + +namespace WebExpress.WebIndex.Test.Fixture +{ + /// + /// Provides a fixture for unit tests that involve the IndexManager and WQL statements for unicode testing. + /// + public class UnitTestIndexFixtureWqlF : UnitTestIndexFixture + { + /// + /// Returns the index manager. + /// + public WebIndex.IndexManager IndexManager { get; } = new IndexManagerTest(); + + /// + /// Returns the test data. + /// + public IEnumerable TestData { get; } = UnitTestIndexTestDocumentFactoryE.GenerateTestData(); + + /// + /// Initializes a new instance of the class. + /// + public UnitTestIndexFixtureWqlF() + { + var context = new IndexContext(); + context.IndexDirectory = Path.Combine(context.IndexDirectory, Path.GetFileNameWithoutExtension(Path.GetRandomFileName())); + + // use reflection to call the protected Initialization method + var method = typeof(IndexManagerTest).GetMethod("Initialization", BindingFlags.Instance | BindingFlags.NonPublic); + method.Invoke(IndexManager, [context]); + + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.ReIndex(TestData); + } + + /// + /// Disposes of the resources used by the current instance. + /// + public override void Dispose() + { + IndexManager.Dispose(); + Directory.Delete(IndexManager.Context.IndexDirectory, true); + } + + /// + /// Executes a wql statement. + /// + /// The wql statement. + /// The WQL parser. + public IWqlStatement ExecuteWql(string wql) + { + return IndexManager.Retrieve(wql); + } + } +} diff --git a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManager.cs b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManager.cs index 7d4d3d6..e5b6895 100644 --- a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManager.cs +++ b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManager.cs @@ -1,4 +1,5 @@ -ο»Ώusing Xunit.Abstractions; +ο»Ώusing System.Reflection; +using Xunit.Abstractions; namespace WebExpress.WebIndex.Test.IndexManager { @@ -37,7 +38,10 @@ protected void Preconditions() var context = new IndexContext(); context.IndexDirectory = Path.Combine(context.IndexDirectory, Path.GetFileNameWithoutExtension(Path.GetRandomFileName())); IndexManager = new IndexManagerTest(); - IndexManager.Initialization(context); + + // use reflection to call the protected Initialization method + var method = typeof(IndexManagerTest).GetMethod("Initialization", BindingFlags.Instance | BindingFlags.NonPublic); + method.Invoke(IndexManager, [context]); Context = context; } diff --git a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryA.cs b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryA.cs index 2b1cce0..d011f01 100644 --- a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryA.cs +++ b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryA.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.IndexManager /// /// Test class for testing the memory-based index manager. /// + [Collection("NonParallelTests")] public class UnitTestIndexManagerMemoryA : UnitTestIndexManager { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. @@ -41,12 +42,16 @@ public void Create() /// /// Tests the reindex function from the index manager. /// - [Fact] - public void ReIndex_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReIndex(string culture) { // preconditions Preconditions(); - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Memory); // test execution IndexManager.ReIndex(Fixture.TestData); @@ -64,12 +69,16 @@ public void ReIndex_En() /// /// Tests the reindex function from the index manager. /// - [Fact] - public async Task ReIndexAsync_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public async Task ReIndexAsync(string culture) { // preconditions Preconditions(); - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Memory); // test execution await IndexManager.ReIndexAsync(Fixture.TestData); @@ -84,75 +93,6 @@ public async Task ReIndexAsync_En() Postconditions(); } - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_De() - { - // preconditions - Preconditions(); - IndexManager.Create(CultureInfo.GetCultureInfo("de"), IndexType.Memory); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve("text ~ 'Helena'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.Equal(4, item.Count()); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_DeDE() - { - // preconditions - Preconditions(); - IndexManager.Create(CultureInfo.GetCultureInfo("de-DE"), IndexType.Memory); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve("text ~ 'Helena'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.Equal(4, item.Count()); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_Fr() - { - // preconditions - Preconditions(); - IndexManager.Create(CultureInfo.GetCultureInfo("fr"), IndexType.Memory); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve("text ~ 'Helena'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.Equal(4, item.Count()); - - // postconditions - Postconditions(); - } - /// /// Tests the removal of a document from the index manager. /// diff --git a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryB.cs b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryB.cs index a4983bb..80f7f90 100644 --- a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryB.cs +++ b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryB.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.IndexManager /// /// Test class for testing the memory-based index manager. /// + [Collection("NonParallelTests")] public class UnitTestIndexManagerMemoryB : UnitTestIndexManager { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. @@ -41,13 +42,17 @@ public void Create() /// /// Tests the reindex function from the index manager. /// - [Fact] - public void ReIndex_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReIndex(string culture) { // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Memory); // test execution IndexManager.ReIndex(Fixture.TestData); @@ -65,13 +70,17 @@ public void ReIndex_En() /// /// Tests the reindex function from the index manager. /// - [Fact] - public async Task ReIndexAsync_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public async Task ReIndexAsync(string culture) { // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Memory); // test execution await IndexManager.ReIndexAsync(Fixture.TestData); @@ -86,78 +95,6 @@ public async Task ReIndexAsync_En() Postconditions(); } - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_De() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de"), IndexType.Memory); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_DeDE() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de-DE"), IndexType.Memory); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_Fr() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("fr"), IndexType.Memory); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - /// /// Tests the removal of a document from the index manager. /// diff --git a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryC.cs b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryC.cs index 243acde..eb5263e 100644 --- a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryC.cs +++ b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryC.cs @@ -1,5 +1,4 @@ -ο»Ώusing System.Diagnostics; -using System.Globalization; +ο»Ώusing System.Globalization; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Test.Fixture; using Xunit.Abstractions; @@ -9,10 +8,11 @@ namespace WebExpress.WebIndex.Test.IndexManager /// /// Test class for testing the memory-based index manager. /// + [Collection("NonParallelTests")] public class UnitTestIndexManagerMemoryC : UnitTestIndexManager { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. @@ -42,13 +42,17 @@ public void Create() /// /// Tests the reindex function from the index manager. /// - [Fact] - public void ReIndex_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReIndex(string culture) { // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Memory); // test execution IndexManager.ReIndex(Fixture.TestData); @@ -64,68 +68,30 @@ public void ReIndex_En() } /// - /// Tests the reindex function from the index manager. + /// Tests the reindex function in a series of tests from the index manager. /// - [Fact] - public async Task ReIndexAsync_En() + [Theory] + [InlineData(100, 100, 100, 15, "en")] + [InlineData(1000, 100, 2000, 15, "en")] + public async Task ReIndexAsync(int itemCount, int wordCount, int vocabulary, int wordLength, string culture) { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); - - // test execution - await IndexManager.ReIndexAsync(Fixture.TestData); - - var wql = IndexManager.Retrieve($"text = '{randomItem.Text}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } + var w = wordCount; + var i = itemCount; + var v = vocabulary; + var l = wordLength; - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_De() - { // preconditions Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de"), IndexType.Memory); - // test execution - IndexManager.ReIndex(Fixture.TestData); + var data = UnitTestIndexTestDocumentFactoryC.GenerateTestData(i, w, v, l); - var wql = IndexManager.Retrieve($"text = '{randomItem.Text}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_DeDE() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de-DE"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Memory); // test execution - IndexManager.ReIndex(Fixture.TestData); + await IndexManager.ReIndexAsync(data); - var wql = IndexManager.Retrieve($"text = '{randomItem.Text}'"); + var randomItem = IndexManager.All().Skip(new Random().Next() % data.Count()).FirstOrDefault(); + var wql = IndexManager.Retrieve($"text = '{randomItem.Text.Split(' ').FirstOrDefault()}'"); Assert.NotNull(wql); var item = wql.Apply(); @@ -135,115 +101,6 @@ public void ReIndex_DeDE() Postconditions(); } - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_Fr() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("fr"), IndexType.Memory); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"text = '{randomItem.Text}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function in a series of tests from the index manager. - /// - [Fact] - public async Task ReIndexAsync_Series() - { - var stopWatch = new Stopwatch(); - - var itemCount = Enumerable.Range(1, 10).Select(x => x * 10000); - var wordCount = Enumerable.Range(1, 1).Select(x => x * 100); - var vocabulary = Enumerable.Range(1, 1).Select(x => x * 20000); - var wordLength = Enumerable.Range(1, 1).Select(x => x * 15); - var file = await Task.Run(() => File.CreateText(Path.Combine(Environment.CurrentDirectory, "memory-reindexasync_series.csv"))); - - Output.WriteLine("item count;wordCount;vocabulary;wordLength;elapsed reindex [hh:mm:ss];elapsed retrieval [ms];size of process mem [MB]"); - file.WriteLine("item count;wordCount;vocabulary;wordLength;elapsed reindex [hh:mm:ss];elapsed retrieval [ms];size of process mem [MB]"); - - foreach (var w in wordCount) - { - foreach (var i in itemCount) - { - foreach (var v in vocabulary) - { - foreach (var l in wordLength) - { - // disabled due to long execution time. activate if necessary. - /** - // preconditions - Preconditions(); - var output = ""; - var data = UnitTestIndexTestDocumentFactoryC.GenerateTestData(i, w, v, l); - var mem = Fixture.GetUsedMemory(); - - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); - - try - { - // preparing for a measurement - stopWatch.Start(); - - // test execution - await IndexManager.ReIndexAsync(data); - - // stop measurement - var elapsedReindex = stopWatch.Elapsed; - stopWatch.Reset(); - - var randomItem = IndexManager.All().Skip(new Random().Next() % data.Count()).FirstOrDefault(); - var wql = IndexManager.Select($"text = '{randomItem.Text.Split(' ').FirstOrDefault()}'"); - Assert.NotNull(wql); - - // preparing for a measurement - stopWatch.Start(); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // stop measurement - var elapsedRetrieval = stopWatch.Elapsed; - stopWatch.Reset(); - - var documentStoreSize = new DirectoryInfo(IndexManager.Context.IndexDirectory).GetFiles("*.wds", SearchOption.AllDirectories).Sum(file => file.Length); - var reverseIndexSize = new DirectoryInfo(IndexManager.Context.IndexDirectory).GetFiles("*.wri", SearchOption.AllDirectories).Sum(file => file.Length); - - output = $"{i};{w};{v};{l};{elapsedReindex:hh\\:mm\\:ss};{(int)Math.Ceiling(elapsedRetrieval.TotalMilliseconds)};{Fixture.GetUsedMemory() - mem}"; - } - catch (Exception ex) - { - output += ex.Message + " " + ex.StackTrace; - } - finally - { - // postconditions - Output.WriteLine(output); - file.WriteLine(output); - file.Flush(); - Postconditions(); - } - /**/ - } - } - } - } - } - /// /// Tests the removal of a document from the index manager. /// diff --git a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryD.cs b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryD.cs index 60353c9..3ef7706 100644 --- a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryD.cs +++ b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryD.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.IndexManager /// /// Test class for testing the memory-based index manager. /// + [Collection("NonParallelTests")] public class UnitTestIndexManagerMemoryD : UnitTestIndexManager { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. @@ -41,13 +42,17 @@ public void Create() /// /// Tests the reindex function from the index manager. /// - [Fact] - public void ReIndex_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReIndex(string culture) { // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Memory); // test execution IndexManager.ReIndex(Fixture.TestData); @@ -65,13 +70,17 @@ public void ReIndex_En() /// /// Tests the reindex function from the index manager. /// - [Fact] - public async Task ReIndexAsync_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public async Task ReIndexAsync(string culture) { // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Memory); // test execution await IndexManager.ReIndexAsync(Fixture.TestData); @@ -86,78 +95,6 @@ public async Task ReIndexAsync_En() Postconditions(); } - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_De() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de"), IndexType.Memory); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"firstname = '{randomItem.FirstName}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_DeDE() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de-DE"), IndexType.Memory); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"firstname = '{randomItem.FirstName}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_Fr() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("fr"), IndexType.Memory); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"firstname = '{randomItem.FirstName}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - /// /// Tests the removal of a document from the index manager. /// diff --git a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryE.cs b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryE.cs index 7175806..c98be05 100644 --- a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryE.cs +++ b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryE.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.IndexManager /// /// Test class for testing the memory-based index manager. /// + [Collection("NonParallelTests")] public class UnitTestIndexManagerMemoryE : UnitTestIndexManager { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. @@ -41,13 +42,17 @@ public void Create() /// /// Tests the reindex function from the index manager. /// - [Fact] - public void ReIndex_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReIndex(string culture) { // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Memory); // test execution IndexManager.ReIndex(Fixture.TestData); @@ -65,13 +70,17 @@ public void ReIndex_En() /// /// Tests the reindex function from the index manager. /// - [Fact] - public async Task ReIndexAsync_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public async Task ReIndexAsync(string culture) { // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Memory); // test execution await IndexManager.ReIndexAsync(Fixture.TestData); @@ -86,78 +95,6 @@ public async Task ReIndexAsync_En() Postconditions(); } - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_De() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de"), IndexType.Memory); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_DeDE() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de-DE"), IndexType.Memory); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_Fr() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("fr"), IndexType.Memory); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - /// /// Tests the removal of a document from the index manager. /// diff --git a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryF.cs b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryF.cs new file mode 100644 index 0000000..b3ab9bd --- /dev/null +++ b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerMemoryF.cs @@ -0,0 +1,342 @@ +ο»Ώusing System.Globalization; +using WebExpress.WebIndex.Test.Document; +using WebExpress.WebIndex.Test.Fixture; +using Xunit.Abstractions; + +namespace WebExpress.WebIndex.Test.IndexManager +{ + /// + /// Test class for testing the memory-based index manager for unicode. + /// + [Collection("NonParallelTests")] + public class UnitTestIndexManagerMemoryF : UnitTestIndexManager + { + /// + /// Initializes a new instance of the class. + /// + /// The log. + /// The test context. + public UnitTestIndexManagerMemoryF(UnitTestIndexFixtureIndexF fixture, ITestOutputHelper output) + : base(fixture, output) + { + } + + /// + /// Tests registering a document in the index manager. + /// + [Fact] + public void Create() + { + // preconditions + Preconditions(); + + // test execution + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + + Assert.NotNull(IndexManager.GetIndexDocument()); + + // postconditions + Postconditions(); + } + + /// + /// Tests the reindex function from the index manager. + /// + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReIndex(string culture) + { + // preconditions + Preconditions(); + var randomItem = Fixture.TestData.LastOrDefault(); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Memory); + + // test execution + IndexManager.ReIndex(Fixture.TestData); + + var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + Assert.NotEmpty(item); + + // postconditions + Postconditions(); + } + + /// + /// Tests the reindex function from the index manager. + /// + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public async Task ReIndexAsync(string culture) + { + // preconditions + Preconditions(); + var randomItem = Fixture.TestData.LastOrDefault(); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Memory); + + // test execution + await IndexManager.ReIndexAsync(Fixture.TestData); + + var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + Assert.NotEmpty(item); + + // postconditions + Postconditions(); + } + + /// + /// Tests the removal of a document from the index manager. + /// + [Fact] + public void Delete() + { + // preconditions + Preconditions(); + var randomItem = Fixture.TestData.LastOrDefault(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.ReIndex(Fixture.TestData); + + var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); + Assert.NotNull(wql); + + var before = wql.Apply().ToList(); + Assert.True(before.Any()); + + // test execution + IndexManager.Delete(randomItem); + + wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); + Assert.NotNull(wql); + + var after = wql.Apply().ToList(); + Assert.True(before.Count - 1 == after.Count); + + // postconditions + Postconditions(); + } + + /// + /// Tests the add function of the index manager. + /// + [Theory] + [InlineData("ED242C79-E41B-4214-BFBC-C4673E87433B", "Aurora")] + [InlineData("A20BC371-10F9-4F43-9DA8-F4B4F0BE26AB", "李明")] + [InlineData("80A78EBB-9819-45AF-BC0F-68E68D0C8C1A", "Sun Leaf Lion 🌞🌿🦁")] + [InlineData("29F34DFD-432D-4315-88C2-CE41F293AC71", "πŸ¦‹πŸŒΌπŸŒ™ Butterfly Flower Moon")] + public void Add(string id, string name) + { + // preconditions + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.ReIndex(Fixture.TestData); + + // test execution + IndexManager.Insert(new UnitTestIndexTestDocumentF() + { + Id = Guid.Parse(id), + Name = name + }); + + var wql = IndexManager.Retrieve($"name = '{name}'"); + var item = wql.Apply(); + + Assert.NotNull(wql); + Assert.Equal(1, item.Count()); + + // postconditions + Postconditions(); + } + + /// + /// Tests the add function of the index manager. + /// + [Theory] + [InlineData("9733A649-1E5E-4B1F-8C6E-9A4B6AB54292", "πŸŒŸπŸ€πŸ‰")] + public void NotAdd(string id, string name) + { + // preconditions + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.ReIndex(Fixture.TestData); + + // test execution + IndexManager.Insert(new UnitTestIndexTestDocumentF() + { + Id = Guid.Parse(id), + Name = name + }); + + var wql = IndexManager.Retrieve($"name = '{name}'"); + var item = wql.Apply(); + + Assert.NotNull(wql); + Assert.Empty(item); + + // postconditions + Postconditions(); + } + + /// + /// Tests the update function of the index manager. + /// + [Fact] + public void Update() + { + // preconditions + Preconditions(); + var randomItem = Fixture.RandomItem; + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.ReIndex(Fixture.TestData); + + // test execution + IndexManager.Update(new UnitTestIndexTestDocumentF() + { + Id = randomItem.Id, + Name = "Aurora" + }); + + var wql = IndexManager.Retrieve("name = 'Aurora'"); + var item = wql.Apply(); + + Assert.NotNull(wql); + Assert.Equal(1, item.Count()); + + // postconditions + Postconditions(); + } + + /// + /// Tests the update function of the index manager. + /// + [Fact] + public async Task UpdateAsync() + { + // preconditions + Preconditions(); + var randomItem = Fixture.RandomItem; + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + await IndexManager.ReIndexAsync(Fixture.TestData); + + // test execution + await IndexManager.UpdateAsync(new UnitTestIndexTestDocumentF() + { + Id = randomItem.Id, + Name = "Aurora" + }); + + var wql = IndexManager.Retrieve("name = 'Aurora'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + Assert.Equal(1, item.Count()); + + // postconditions + Postconditions(); + } + + /// + /// Tests removing a document on the index manager. + /// + [Fact] + public void Clear() + { + // preconditions + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.ReIndex(Fixture.TestData); + + var documents = IndexManager.All(); + + Assert.NotNull(documents); + Assert.True(documents.Any()); + + // test execution + IndexManager.Clear(); + + documents = IndexManager.All(); + + Assert.NotNull(documents); + Assert.False(documents.Any()); + + // postconditions + Postconditions(); + } + + /// + /// Return all entries of the index manager. + /// + [Fact] + public void All() + { + // preconditions + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.ReIndex(Fixture.TestData); + + // test execution + var all = IndexManager.All(); + + Assert.NotEmpty(all); + + // postconditions + Postconditions(); + } + + /// + /// Tests get a document from the index manager. + /// + [Fact] + public void GetDocument() + { + // preconditions + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + + // test execution + var document = IndexManager.GetIndexDocument(); + Assert.NotNull(document); + Assert.True(document.GetType() == typeof(IndexDocument)); + + // postconditions + Postconditions(); + } + + /// + /// Tests get a document from the index manager. + /// + [Fact] + public void GetDocument_Not() + { + // preconditions + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Memory); + + // test execution + var document = IndexManager.GetIndexDocument(); + Assert.Null(document); + + // postconditions + Postconditions(); + } + } +} diff --git a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageA.cs b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageA.cs index 3534465..86a453e 100644 --- a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageA.cs +++ b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageA.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.IndexManager /// /// Test class for testing the storage-based index manager. /// + [Collection("NonParallelTests")] public class UnitTestIndexManagerStorageA : UnitTestIndexManager { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. @@ -41,12 +42,16 @@ public void Create() /// /// Tests the reindex function from the index manager. /// - [Fact] - public void ReIndex_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReIndex(string culture) { // preconditions Preconditions(); - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); // test execution IndexManager.ReIndex(Fixture.TestData); @@ -64,12 +69,16 @@ public void ReIndex_En() /// /// Tests the reindex function from the index manager. /// - [Fact] - public async Task ReIndexAsync_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public async Task ReIndexAsync(string culture) { // preconditions Preconditions(); - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); // test execution await IndexManager.ReIndexAsync(Fixture.TestData); @@ -85,16 +94,14 @@ public async Task ReIndexAsync_En() } /// - /// Tests the reindex function from the index manager. + /// Tests the removal of a document from the index manager. /// [Fact] - public void ReIndex_De() + public void Delete() { // preconditions Preconditions(); - IndexManager.Create(CultureInfo.GetCultureInfo("de"), IndexType.Storage); - - // test execution + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); IndexManager.ReIndex(Fixture.TestData); var wql = IndexManager.Retrieve("text ~ 'Helena'"); @@ -103,109 +110,99 @@ public void ReIndex_De() var item = wql.Apply(); Assert.Equal(4, item.Count()); - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_DeDE() - { - // preconditions - Preconditions(); - IndexManager.Create(CultureInfo.GetCultureInfo("de-DE"), IndexType.Storage); - // test execution - IndexManager.ReIndex(Fixture.TestData); + IndexManager.Delete(Fixture.TestData[0]); - var wql = IndexManager.Retrieve("text ~ 'Helena'"); + wql = IndexManager.Retrieve("text ~ 'Helena'"); Assert.NotNull(wql); - var item = wql.Apply(); - Assert.Equal(4, item.Count()); + item = wql.Apply(); + Assert.Equal(3, item.Count()); // postconditions Postconditions(); } /// - /// Tests the reindex function from the index manager. + /// Tests the add function of the index manager. /// [Fact] - public void ReIndex_Fr() + public void Add() { // preconditions Preconditions(); - IndexManager.Create(CultureInfo.GetCultureInfo("fr"), IndexType.Storage); - - // test execution + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); IndexManager.ReIndex(Fixture.TestData); - var wql = IndexManager.Retrieve("text ~ 'Helena'"); - Assert.NotNull(wql); + // test execution + IndexManager.Insert(new UnitTestIndexTestDocumentA() + { + Id = Guid.Parse("ED242C79-E41B-4214-BFBC-C4673E87433B"), + Text = "Hello Aurora!" + }); + var wql = IndexManager.Retrieve("text ~ 'Aurora'"); var item = wql.Apply(); - Assert.Equal(4, item.Count()); + + Assert.NotNull(wql); + Assert.Equal(1, item.Count()); // postconditions Postconditions(); } /// - /// Tests the removal of a document from the index manager. + /// Tests the register wql function of the index manager. /// [Fact] - public void Delete() + public void RegisterWqlFunction() { // preconditions Preconditions(); IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve("text ~ 'Helena'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.Equal(4, item.Count()); + IndexManager.Insert(new UnitTestIndexTestDocumentA() + { + Id = Guid.Parse("ED242C79-E41B-4214-BFBC-C4673E87433B"), + Text = "abc" + }); // test execution - IndexManager.Delete(Fixture.TestData[0]); + IndexManager.RegisterWqlFunction>(); - wql = IndexManager.Retrieve("text ~ 'Helena'"); + var functions = IndexManager.WqlFunctions; + Assert.NotEmpty(functions); + + var wql = IndexManager.Retrieve("text ~ 'abc'"); Assert.NotNull(wql); - item = wql.Apply(); - Assert.Equal(3, item.Count()); + var item = wql.Apply(); + Assert.Equal("abc", item.FirstOrDefault()?.Text); // postconditions Postconditions(); } /// - /// Tests the add function of the index manager. + /// Tests the remove wql function of the index manager. /// [Fact] - public void Add() + public void RemoveWqlFunction() { // preconditions Preconditions(); IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); - IndexManager.ReIndex(Fixture.TestData); - - // test execution IndexManager.Insert(new UnitTestIndexTestDocumentA() { Id = Guid.Parse("ED242C79-E41B-4214-BFBC-C4673E87433B"), - Text = "Hello Aurora!" + Text = "abc" }); - var wql = IndexManager.Retrieve("text ~ 'Aurora'"); - var item = wql.Apply(); + IndexManager.RegisterWqlFunction>(); + Assert.NotEmpty(IndexManager.WqlFunctions); - Assert.NotNull(wql); - Assert.Equal(1, item.Count()); + // test execution + IndexManager.RemoveWqlFunction>(); + Assert.Empty(IndexManager.WqlFunctions); // postconditions Postconditions(); @@ -370,5 +367,39 @@ public void GetDocument_Not() // postconditions Postconditions(); } + + /// + /// Tests the close and open function from the index manager. + /// + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReOpen(string culture) + { + // preconditions + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + IndexManager.ReIndex(Fixture.TestData); + var wql = IndexManager.Retrieve("text ~ 'Helena'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + var count = item.Count(); + + // test execution + IndexManager.Close(); + + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + wql = IndexManager.Retrieve("text ~ 'Helena'"); + Assert.NotNull(wql); + + item = wql.Apply(); + Assert.Equal(count, item.Count()); + + // postconditions + Postconditions(); + } } } diff --git a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageB.cs b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageB.cs index eca873f..fda07ac 100644 --- a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageB.cs +++ b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageB.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.IndexManager /// /// Test class for testing the storage-based index manager. /// + [Collection("NonParallelTests")] public class UnitTestIndexManagerStorageB : UnitTestIndexManager { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. @@ -41,13 +42,17 @@ public void Create() /// /// Tests the reindex function from the index manager. /// - [Fact] - public void ReIndex_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReIndex(string culture) { // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); // test execution IndexManager.ReIndex(Fixture.TestData); @@ -65,13 +70,17 @@ public void ReIndex_En() /// /// Tests the reindex function from the index manager. /// - [Fact] - public async Task ReIndexAsync_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public async Task ReIndexAsync(string culture) { // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); // test execution await IndexManager.ReIndexAsync(Fixture.TestData); @@ -91,7 +100,7 @@ public async Task ReIndexAsync_En() /// Tests the reindex function from the index manager. /// [Fact] - public async Task ReIndexAsyncCancel_En() + public async Task ReIndexAsyncCancel() { // preconditions Preconditions(); @@ -121,78 +130,6 @@ public async Task ReIndexAsyncCancel_En() Postconditions(); } - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_De() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de"), IndexType.Storage); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_DeDE() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de-DE"), IndexType.Storage); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_Fr() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("fr"), IndexType.Storage); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - /// /// Tests the removal of a document from the index manager. /// @@ -209,7 +146,7 @@ public void Delete() Assert.NotNull(wql); var before = wql.Apply().ToList(); - Assert.True(before.Any()); + Assert.NotEmpty(before); // test execution IndexManager.Delete(randomItem); @@ -401,5 +338,67 @@ public void GetDocument_Not() // postconditions Postconditions(); } + + /// + /// Tests the retrieve function in a series of tests from the index manager. + /// + [Theory] + [InlineData("name = 'Name_3'", "en", "Name_3")] + [InlineData("Summary = 'Name_3'", "en", "Name_3")] + [InlineData("Price = 3", "en", "Name_3")] + [InlineData("Price = '3'", "en", "Name_3")] + [InlineData("Adress.Street = 3", "en", "Name_3")] + [InlineData("Adress.Country = usa", "en", "Name_3")] + public void Retrieve(string wqlString, string cultureString, string expected) + { + // preconditions + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo(cultureString), IndexType.Storage); + IndexManager.ReIndex(Fixture.TestData); + var wql = IndexManager.Retrieve(wqlString); + Assert.NotNull(wql); + + // test execution + var items = wql.Apply(); + Assert.Contains(expected, items.Select(x => x.Name.ToString())); + + // postconditions + Postconditions(); + } + + /// + /// Tests the close and open function from the index manager. + /// + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReOpen(string culture) + { + // preconditions + Preconditions(); + var randomItem = Fixture.RandomItem; + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + IndexManager.ReIndex(Fixture.TestData); + var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + var count = item.Count(); + + // test execution + IndexManager.Close(); + + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); + Assert.NotNull(wql); + + item = wql.Apply(); + Assert.Equal(count, item.Count()); + + // postconditions + Postconditions(); + } } } diff --git a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageC.cs b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageC.cs index 2a50f26..d2c190c 100644 --- a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageC.cs +++ b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageC.cs @@ -1,5 +1,4 @@ -ο»Ώusing System.Diagnostics; -using System.Globalization; +ο»Ώusing System.Globalization; using WebExpress.WebIndex.Storage; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Test.Fixture; @@ -12,6 +11,7 @@ namespace WebExpress.WebIndex.Test.IndexManager /// /// The log. /// The test context. + [Collection("NonParallelTests")] public class UnitTestIndexManagerStorageC(UnitTestIndexFixtureIndexC fixture, ITestOutputHelper output) : UnitTestIndexManager(fixture, output) { /// @@ -35,13 +35,17 @@ public void Create() /// /// Tests the reindex function from the index manager. /// - [Fact] - public void ReIndex_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReIndex(string culture) { // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); // test execution IndexManager.ReIndex(Fixture.TestData); @@ -57,92 +61,39 @@ public void ReIndex_En() } /// - /// Tests the reindex function from the index manager. + /// Tests the reindex function in a series of tests from the index manager. /// - [Fact] - public async Task ReIndexAsync_En() + [Theory] + [InlineData(100, 100, 100, 15, "en")] + [InlineData(1000, 100, 2000, 15, "en")] + public async Task ReIndexAsync(int itemCount, int wordCount, int vocabulary, int wordLength, string culture) { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); - - // test execution - await IndexManager.ReIndexAsync(Fixture.TestData); - - var wql = IndexManager.Retrieve($"text = '{randomItem.Text}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); + var w = wordCount; + var i = itemCount; + var v = vocabulary; + var l = wordLength; + var maxCachedSegmentsRange = 50000u; + var bufferSizeRange = Math.Pow(2, 12); + var path = Path.Combine(Environment.CurrentDirectory, "storage-reindexasync_series.csv"); + var exists = File.Exists(path); - // postconditions - Postconditions(); - } + var data = UnitTestIndexTestDocumentFactoryC.GenerateTestData(i, w, v, l); + var randomItem = default(UnitTestIndexTestDocumentC); + var mem = Fixture.GetUsedMemory(); + var tasks = new List(); - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_De() - { // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de"), IndexType.Storage); + IndexStorageBuffer.MaxCachedSegments = maxCachedSegmentsRange; + IndexStorageFile.BufferSize = (uint)bufferSizeRange; - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"text = '{randomItem.Text.Split(' ').FirstOrDefault()}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_DeDE() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de-DE"), IndexType.Storage); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"text = '{randomItem.Text.Split(' ').FirstOrDefault()}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_Fr() - { - // preconditions Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("fr"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); // test execution - IndexManager.ReIndex(Fixture.TestData); + await IndexManager.ReIndexAsync(data); - var wql = IndexManager.Retrieve($"text = '{randomItem.Text.Split(' ').FirstOrDefault()}'"); + randomItem ??= IndexManager.All().Skip(new Random().Next() % data.Count()).FirstOrDefault(); + var wql = IndexManager.Retrieve($"text ~ '{randomItem.Text.Split(' ').FirstOrDefault()}'"); Assert.NotNull(wql); var item = wql.Apply(); @@ -152,212 +103,6 @@ public void ReIndex_Fr() Postconditions(); } - /// - /// Tests the reindex function in a series of tests from the index manager. - /// - [Fact] - public void ReIndex_Series() - { - var stopWatch = new Stopwatch(); - - var itemCount = Enumerable.Range(1, 1).Select(x => x * 100); - var wordCount = Enumerable.Range(1, 1).Select(x => x * 100); - var vocabulary = Enumerable.Range(1, 1).Select(x => x * 20000); - var wordLength = Enumerable.Range(1, 1).Select(x => x * 15); - var maxCachedSegments = Enumerable.Range(5, 1).Select(x => x * 10000); - var bufferSize = Enumerable.Range(12, 1).Select(x => Math.Pow(2, x)); - var path = Path.Combine(Environment.CurrentDirectory, "storage-reindex_series.csv"); - var exists = File.Exists(path); - - if (!exists) - { - File.AppendAllText(path, "timestamp;item count;wordCount;vocabulary;wordLength;max cached segments;buffer size;elapsed reindex [hh:mm:ss];elapsed retrieval [ms];size of document store [MB];size of reverse index [MB];βˆ‘ storage space [MB];size of process mem [MB]" + Environment.NewLine); - } - - foreach (var w in wordCount) - { - foreach (var i in itemCount) - { - foreach (var v in vocabulary) - { - foreach (var l in wordLength) - { - var data = UnitTestIndexTestDocumentFactoryC.GenerateTestData(i, w, v, l); - var randomItem = default(UnitTestIndexTestDocumentC); - var mem = Fixture.GetUsedMemory(); - - foreach (var m in maxCachedSegments) - { - foreach (var b in bufferSize) - { - // preconditions - IndexStorageBuffer.MaxCachedSegments = (uint)m; - IndexStorageFile.BufferSize = (uint)b; - - Preconditions(); - var output = ""; - - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); - IndexManager.Retrieve($"text = 'xyz'").Apply(); - - try - { - // preparing for a measurement - stopWatch.Start(); - - // test execution - IndexManager.ReIndex(data); - - // stop measurement - var elapsedReindex = stopWatch.Elapsed; - stopWatch.Reset(); - - randomItem ??= IndexManager.All().Skip(new Random().Next() % data.Count()).FirstOrDefault(); - var wql = IndexManager.Retrieve($"text ~ '{randomItem.Text.Split(' ').FirstOrDefault()}'"); - Assert.NotNull(wql); - - // preparing for a measurement - stopWatch.Start(); - - var item = wql.Apply(); - - // stop measurement - var elapsedRetrieval = stopWatch.Elapsed; - stopWatch.Reset(); - - Assert.NotEmpty(item); - - IndexManager.Dispose(); - - var documentStoreSize = new DirectoryInfo(IndexManager.Context.IndexDirectory).GetFiles("*.wds", SearchOption.AllDirectories).Sum(file => file.Length); - var reverseIndexSize = new DirectoryInfo(IndexManager.Context.IndexDirectory).GetFiles("*.wri", SearchOption.AllDirectories).Sum(file => file.Length); - - output = $"{DateTime.Now};{i};{w};{v};{l};{m};{b};{elapsedReindex:hh\\:mm\\:ss};{(int)Math.Ceiling(elapsedRetrieval.TotalMilliseconds)};{Math.Round((double)documentStoreSize / 1024 / 1024, 2)};{Math.Round((double)reverseIndexSize / 1024 / 1024, 2)};{Math.Round((double)(documentStoreSize + reverseIndexSize) / 1024 / 1024, 2)};{Fixture.GetUsedMemory() - mem}"; - } - catch (Exception ex) - { - Output.WriteLine(ex.Message + " " + ex.StackTrace); - throw; - } - finally - { - // postconditions - File.AppendAllText(path, output + Environment.NewLine); - Postconditions(); - } - - Thread.Sleep(5000); - } - } - } - } - } - } - } - - /// - /// Tests the reindex function in a series of tests from the index manager. - /// - [Fact] - public async Task ReIndexAsync_Series() - { - var stopWatch = new Stopwatch(); - - var itemCount = Enumerable.Range(1, 1).Select(x => x * 10000); - var wordCount = Enumerable.Range(1, 1).Select(x => x * 100); - var vocabulary = Enumerable.Range(1, 1).Select(x => x * 20000); - var wordLength = Enumerable.Range(1, 1).Select(x => x * 15); - var maxCachedSegments = Enumerable.Range(5, 1).Select(x => x * 10000); - var bufferSize = Enumerable.Range(12, 1).Select(x => Math.Pow(2, x)); - var path = Path.Combine(Environment.CurrentDirectory, "storage-reindexasync_series.csv"); - var exists = File.Exists(path); - - if (!exists) - { - File.AppendAllText(path, "timestamp;item count;wordCount;vocabulary;wordLength;max cached segments;buffer size;elapsed reindex [hh:mm:ss];elapsed retrieval [ms];size of document store [MB];size of reverse index [MB];βˆ‘ storage space [MB];size of process mem [MB]" + Environment.NewLine); - } - - foreach (var w in wordCount) - { - foreach (var i in itemCount) - { - foreach (var v in vocabulary) - { - foreach (var l in wordLength) - { - var data = UnitTestIndexTestDocumentFactoryC.GenerateTestData(i, w, v, l); - var randomItem = default(UnitTestIndexTestDocumentC); - var mem = Fixture.GetUsedMemory(); - - foreach (var m in maxCachedSegments) - { - foreach (var b in bufferSize) - { - // preconditions - IndexStorageBuffer.MaxCachedSegments = (uint)m; - IndexStorageFile.BufferSize = (uint)b; - - Preconditions(); - var output = ""; - - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); - IndexManager.Retrieve($"text = 'xyz'").Apply(); - - try - { - // preparing for a measurement - stopWatch.Start(); - - // test execution - await IndexManager.ReIndexAsync(data); - - // stop measurement - var elapsedReindex = stopWatch.Elapsed; - stopWatch.Reset(); - - randomItem ??= IndexManager.All().Skip(new Random().Next() % data.Count()).FirstOrDefault(); - var wql = IndexManager.Retrieve($"text ~ '{randomItem.Text.Split(' ').FirstOrDefault()}'"); - Assert.NotNull(wql); - - // preparing for a measurement - stopWatch.Start(); - - var item = wql.Apply(); - - // stop measurement - var elapsedRetrieval = stopWatch.Elapsed; - stopWatch.Reset(); - - Assert.NotEmpty(item); - - IndexManager.Dispose(); - - var documentStoreSize = new DirectoryInfo(IndexManager.Context.IndexDirectory).GetFiles("*.wds", SearchOption.AllDirectories).Sum(file => file.Length); - var reverseIndexSize = new DirectoryInfo(IndexManager.Context.IndexDirectory).GetFiles("*.wri", SearchOption.AllDirectories).Sum(file => file.Length); - - output = $"{DateTime.Now};{i};{w};{v};{l};{m};{b};{elapsedReindex:hh\\:mm\\:ss};{(int)Math.Ceiling(elapsedRetrieval.TotalMilliseconds)};{Math.Round((double)documentStoreSize / 1024 / 1024, 2)};{Math.Round((double)reverseIndexSize / 1024 / 1024, 2)};{Math.Round((double)(documentStoreSize + reverseIndexSize) / 1024 / 1024, 2)};{Fixture.GetUsedMemory() - mem}"; - } - catch (Exception ex) - { - Output.WriteLine(ex.Message + " " + ex.StackTrace); - throw; - } - finally - { - // postconditions - File.AppendAllText(path, output + Environment.NewLine); - Postconditions(); - } - - Thread.Sleep(5000); - } - } - } - } - } - } - } - /// /// Tests the removal of a document from the index manager. /// @@ -566,5 +311,144 @@ public void GetDocument_Not() // postconditions Postconditions(); } + + /// + /// Tests the close and open function from the index manager. + /// + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReOpen(string culture) + { + // preconditions + Preconditions(); + var randomItem = Fixture.RandomItem; + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + IndexManager.ReIndex(Fixture.TestData); + var wql = IndexManager.Retrieve($"text = '{randomItem.Text.Split(' ').FirstOrDefault()}'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + var count = item.Count(); + + // test execution + IndexManager.Close(); + + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + wql = IndexManager.Retrieve($"text = '{randomItem.Text.Split(' ').FirstOrDefault()}'"); + Assert.NotNull(wql); + + item = wql.Apply(); + Assert.Equal(count, item.Count()); + + // postconditions + Postconditions(); + } + + /// + /// Tests the retrieve function in a series of tests from the index manager. + /// + [Theory] + [InlineData(100, 100, 100, 15, "en")] + [InlineData(1000, 100, 2000, 15, "en")] + public void Retrieve(int itemCount, int wordCount, int vocabulary, int wordLength, string culture) + { + var w = wordCount; + var i = itemCount; + var v = vocabulary; + var l = wordLength; + var maxCachedSegmentsRange = 50000u; + var bufferSizeRange = Math.Pow(2, 12); + var path = Path.Combine(Environment.CurrentDirectory, "storage-reindexasync_series.csv"); + var exists = File.Exists(path); + + var data = UnitTestIndexTestDocumentFactoryC.GenerateTestData(i, w, v, l); + var randomItem = default(UnitTestIndexTestDocumentC); + var mem = Fixture.GetUsedMemory(); + var tasks = new List(); + + // preconditions + IndexStorageBuffer.MaxCachedSegments = maxCachedSegmentsRange; + IndexStorageFile.BufferSize = (uint)bufferSizeRange; + + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + IndexManager.ReIndex(data); + randomItem ??= IndexManager.All().Skip(new Random().Next() % data.Count()).FirstOrDefault(); + + for (int t = 0; t < 25; t++) + { + tasks.Add(Task.Run(() => + { + // test execution + var wql = IndexManager.Retrieve($"text ~ '{randomItem.Text.Split(' ').FirstOrDefault()}'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + Assert.NotEmpty(item); + + return Task.CompletedTask; + })); + } + + Task.WhenAll(tasks); + + // postconditions + Postconditions(); + } + + /// + /// Tests the retrieve function in a series of tests from the index manager. + /// + [Theory] + [InlineData(100, 100, 100, 15, "en")] + [InlineData(1000, 100, 2000, 15, "en")] + public async Task RetrieveAsync(int itemCount, int wordCount, int vocabulary, int wordLength, string culture) + { + var w = wordCount; + var i = itemCount; + var v = vocabulary; + var l = wordLength; + var maxCachedSegmentsRange = 50000u; + var bufferSizeRange = Math.Pow(2, 12); + var path = Path.Combine(Environment.CurrentDirectory, "storage-reindexasync_series.csv"); + var exists = File.Exists(path); + + var data = UnitTestIndexTestDocumentFactoryC.GenerateTestData(i, w, v, l); + var randomItem = default(UnitTestIndexTestDocumentC); + var mem = Fixture.GetUsedMemory(); + var tasks = new List(); + + // preconditions + IndexStorageBuffer.MaxCachedSegments = maxCachedSegmentsRange; + IndexStorageFile.BufferSize = (uint)bufferSizeRange; + + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + await IndexManager.ReIndexAsync(data); + randomItem ??= IndexManager.All().Skip(new Random().Next() % data.Count()).FirstOrDefault(); + + for (int t = 0; t < 25; t++) + { + tasks.Add(await Task.Run(async () => + { + // test execution + var wql = await IndexManager.RetrieveAsync($"text ~ '{randomItem.Text.Split(' ').FirstOrDefault()}'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + Assert.NotEmpty(item); + + return Task.CompletedTask; + })); + } + + await Task.WhenAll(tasks); + + // postconditions + Postconditions(); + } } } diff --git a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageD.cs b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageD.cs index ead6943..717a055 100644 --- a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageD.cs +++ b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageD.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.IndexManager /// /// Test class for testing the storage-based index manager. /// + [Collection("NonParallelTests")] public class UnitTestIndexManagerStorageD : UnitTestIndexManager { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. @@ -41,13 +42,17 @@ public void Create() /// /// Tests the reindex function from the index manager. /// - [Fact] - public void ReIndex_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReIndex(string culture) { // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); // test execution IndexManager.ReIndex(Fixture.TestData); @@ -64,14 +69,17 @@ public void ReIndex_En() /// /// Tests the reindex function from the index manager. - /// - [Fact] - public async Task ReIndexAsync_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public async Task ReIndexAsync(string culture) { // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); // test execution await IndexManager.ReIndexAsync(Fixture.TestData); @@ -86,78 +94,6 @@ public async Task ReIndexAsync_En() Postconditions(); } - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_De() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de"), IndexType.Storage); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"firstname = '{randomItem.FirstName}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_DeDE() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de-DE"), IndexType.Storage); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"firstname = '{randomItem.FirstName}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_Fr() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("fr"), IndexType.Storage); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"firstname = '{randomItem.FirstName}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - /// /// Tests the removal of a document from the index manager. /// @@ -366,5 +302,40 @@ public void GetDocument_Not() // postconditions Postconditions(); } + + /// + /// Tests the close and open function from the index manager. + /// + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReOpen(string culture) + { + // preconditions + Preconditions(); + var randomItem = Fixture.RandomItem; + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + IndexManager.ReIndex(Fixture.TestData); + var wql = IndexManager.Retrieve($"firstname = '{randomItem.FirstName}'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + var count = item.Count(); + + // test execution + IndexManager.Close(); + + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + wql = IndexManager.Retrieve($"firstname = '{randomItem.FirstName}'"); + Assert.NotNull(wql); + + item = wql.Apply(); + Assert.Equal(count, item.Count()); + + // postconditions + Postconditions(); + } } } diff --git a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageE.cs b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageE.cs index 9757d5e..4f1591c 100644 --- a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageE.cs +++ b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageE.cs @@ -8,10 +8,11 @@ namespace WebExpress.WebIndex.Test.IndexManager /// /// Test class for testing the storage-based index manager. /// + [Collection("NonParallelTests")] public class UnitTestIndexManagerStorageE : UnitTestIndexManager { /// - /// Constructor + /// Initializes a new instance of the class. /// /// The log. /// The test context. @@ -41,13 +42,17 @@ public void Create() /// /// Tests the reindex function from the index manager. /// - [Fact] - public void ReIndex_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReIndex(string culture) { // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); // test execution IndexManager.ReIndex(Fixture.TestData); @@ -65,13 +70,17 @@ public void ReIndex_En() /// /// Tests the reindex function from the index manager. /// - [Fact] - public async Task ReIndexAsync_En() + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public async Task ReIndexAsync(string culture) { // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); // test execution await IndexManager.ReIndexAsync(Fixture.TestData); @@ -86,78 +95,6 @@ public async Task ReIndexAsync_En() Postconditions(); } - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_De() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de"), IndexType.Storage); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_DeDE() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("de-DE"), IndexType.Storage); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - - /// - /// Tests the reindex function from the index manager. - /// - [Fact] - public void ReIndex_Fr() - { - // preconditions - Preconditions(); - var randomItem = Fixture.RandomItem; - IndexManager.Create(CultureInfo.GetCultureInfo("fr"), IndexType.Storage); - - // test execution - IndexManager.ReIndex(Fixture.TestData); - - var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); - Assert.NotNull(wql); - - var item = wql.Apply(); - Assert.NotEmpty(item); - - // postconditions - Postconditions(); - } - /// /// Tests the removal of a document from the index manager. /// @@ -254,7 +191,7 @@ public async Task UpdateAsync() { // preconditions Preconditions(); - var randomItem = Fixture.RandomItem; + var randomItem = Fixture.TestData.LastOrDefault(); IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); await IndexManager.ReIndexAsync(Fixture.TestData); @@ -366,5 +303,40 @@ public void GetDocument_Not() // postconditions Postconditions(); } + + /// + /// Tests the close and open function from the index manager. + /// + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReOpen(string culture) + { + // preconditions + Preconditions(); + var randomItem = Fixture.RandomItem; + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + IndexManager.ReIndex(Fixture.TestData); + var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + var count = item.Count(); + + // test execution + IndexManager.Close(); + + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); + Assert.NotNull(wql); + + item = wql.Apply(); + Assert.Equal(count, item.Count()); + + // postconditions + Postconditions(); + } } } diff --git a/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageF.cs b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageF.cs new file mode 100644 index 0000000..52f6bcf --- /dev/null +++ b/src/WebExpress.WebIndex.Test/IndexManager/UnitTestIndexManagerStorageF.cs @@ -0,0 +1,377 @@ +ο»Ώusing System.Globalization; +using WebExpress.WebIndex.Test.Document; +using WebExpress.WebIndex.Test.Fixture; +using Xunit.Abstractions; + +namespace WebExpress.WebIndex.Test.IndexManager +{ + /// + /// Test class for testing the storage-based index manager for unicode. + /// + [Collection("NonParallelTests")] + public class UnitTestIndexManagerStorageF : UnitTestIndexManager + { + /// + /// Initializes a new instance of the class. + /// + /// The log. + /// The test context. + public UnitTestIndexManagerStorageF(UnitTestIndexFixtureIndexF fixture, ITestOutputHelper output) + : base(fixture, output) + { + } + + /// + /// Tests registering a document in the index manager. + /// + [Fact] + public void Create() + { + // preconditions + Preconditions(); + + // test execution + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + + Assert.NotNull(IndexManager.GetIndexDocument()); + + // postconditions + Postconditions(); + } + + /// + /// Tests the reindex function from the index manager. + /// + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReIndex(string culture) + { + // preconditions + Preconditions(); + var randomItem = Fixture.TestData?.LastOrDefault(); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + + // test execution + IndexManager.ReIndex(Fixture.TestData); + + var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + Assert.NotEmpty(item); + + // postconditions + Postconditions(); + } + + /// + /// Tests the reindex function from the index manager. + /// + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public async Task ReIndexAsync(string culture) + { + // preconditions + Preconditions(); + var randomItem = Fixture.TestData?.LastOrDefault(); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + + // test execution + await IndexManager.ReIndexAsync(Fixture.TestData); + + var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + Assert.NotEmpty(item); + + // postconditions + Postconditions(); + } + + /// + /// Tests the removal of a document from the index manager. + /// + [Fact] + public void Delete() + { + // preconditions + Preconditions(); + var randomItem = Fixture.TestData.LastOrDefault(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.ReIndex(Fixture.TestData); + + var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); + Assert.NotNull(wql); + + var before = wql.Apply().ToList(); + Assert.True(before.Any()); + + // test execution + IndexManager.Delete(randomItem); + + wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); + Assert.NotNull(wql); + + var after = wql.Apply().ToList(); + Assert.Equal(before.Count - 1, after.Count); + + // postconditions + Postconditions(); + } + + /// + /// Tests the add function of the index manager. + /// + [Theory] + [InlineData("ED242C79-E41B-4214-BFBC-C4673E87433B", "Aurora")] + [InlineData("A20BC371-10F9-4F43-9DA8-F4B4F0BE26AB", "李明")] + [InlineData("80A78EBB-9819-45AF-BC0F-68E68D0C8C1A", "Sun Leaf Lion 🌞🌿🦁")] + [InlineData("29F34DFD-432D-4315-88C2-CE41F293AC71", "πŸ¦‹πŸŒΌπŸŒ™ Butterfly Flower Moon")] + public void Add(string id, string name) + { + // preconditions + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.ReIndex(Fixture.TestData); + + // test execution + IndexManager.Insert(new UnitTestIndexTestDocumentF() + { + Id = Guid.Parse(id), + Name = name + }); + + var wql = IndexManager.Retrieve($"name = '{name}'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + Assert.Equal(1, item.Count()); + + // postconditions + Postconditions(); + } + + /// + /// Tests the add function of the index manager. + /// + [Theory] + [InlineData("9733A649-1E5E-4B1F-8C6E-9A4B6AB54292", "πŸŒŸπŸ€πŸ‰")] + public void NotAdd(string id, string name) + { + // preconditions + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.ReIndex(Fixture.TestData); + + // test execution + IndexManager.Insert(new UnitTestIndexTestDocumentF() + { + Id = Guid.Parse(id), + Name = name + }); + + var wql = IndexManager.Retrieve($"name = '{name}'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + Assert.Empty(item); + + // postconditions + Postconditions(); + } + + /// + /// Tests the update function of the index manager. + /// + [Fact] + public void Update() + { + // preconditions + Preconditions(); + var randomItem = Fixture.TestData?.LastOrDefault(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.ReIndex(Fixture.TestData); + + // test execution + IndexManager.Update(new UnitTestIndexTestDocumentF() + { + Id = randomItem.Id, + Name = "Aurora" + }); + + var wql = IndexManager.Retrieve("name = 'Aurora'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + Assert.Equal(1, item.Count()); + + // postconditions + Postconditions(); + } + + /// + /// Tests the update function of the index manager. + /// + [Fact] + public async Task UpdateAsync() + { + // preconditions + Preconditions(); + var randomItem = Fixture.TestData?.LastOrDefault(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + await IndexManager.ReIndexAsync(Fixture.TestData); + + // test execution + await IndexManager.UpdateAsync(new UnitTestIndexTestDocumentF() + { + Id = randomItem.Id, + Name = "Aurora" + }); + + var wql = IndexManager.Retrieve("name = 'Aurora'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + Assert.Equal(1, item.Count()); + + // postconditions + Postconditions(); + } + + /// + /// Tests removing a document on the index manager. + /// + [Fact] + public void Clear() + { + // preconditions + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.ReIndex(Fixture.TestData); + + var documents = IndexManager.All(); + + Assert.NotNull(documents); + Assert.True(documents.Any()); + + // test execution + IndexManager.Clear(); + + documents = IndexManager.All(); + + Assert.NotNull(documents); + Assert.False(documents.Any()); + + // postconditions + Postconditions(); + } + + /// + /// Return all entries of the index manager. + /// + [Fact] + public void All() + { + // preconditions + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.ReIndex(Fixture.TestData); + + // test execution + var all = IndexManager.All(); + + Assert.True(all.Select(x => x.Id).OrderBy(x => x).SequenceEqual(Fixture.TestData.Select(x => x.Id).OrderBy(x => x))); + + // postconditions + Postconditions(); + } + + /// + /// Tests get a document from the index manager. + /// + [Fact] + public void GetDocument() + { + // preconditions + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + + // test execution + var document = IndexManager.GetIndexDocument(); + Assert.NotNull(document); + Assert.True(document.GetType() == typeof(IndexDocument)); + + // postconditions + Postconditions(); + } + + /// + /// Tests get a document from the index manager. + /// + [Fact] + public void GetDocument_Not() + { + // preconditions + Preconditions(); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + IndexManager.Create(CultureInfo.GetCultureInfo("en"), IndexType.Storage); + + // test execution + var document = IndexManager.GetIndexDocument(); + Assert.Null(document); + + // postconditions + Postconditions(); + } + + /// + /// Tests the close and open function from the index manager. + /// + [Theory] + [InlineData("en")] + [InlineData("de")] + [InlineData("de-DE")] + [InlineData("fr")] + public void ReOpen(string culture) + { + // preconditions + Preconditions(); + var randomItem = Fixture.TestData?.LastOrDefault(); + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + IndexManager.ReIndex(Fixture.TestData); + var wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); + Assert.NotNull(wql); + + var item = wql.Apply(); + var count = item.Count(); + + // test execution + IndexManager.Close(); + + IndexManager.Create(CultureInfo.GetCultureInfo(culture), IndexType.Storage); + wql = IndexManager.Retrieve($"name = '{randomItem.Name}'"); + Assert.NotNull(wql); + + item = wql.Apply(); + Assert.Equal(count, item.Count()); + + // postconditions + Postconditions(); + } + } +} diff --git a/src/WebExpress.WebIndex.Test/IndexManagerTest.cs b/src/WebExpress.WebIndex.Test/IndexManagerTest.cs index ac7a569..a7488df 100644 --- a/src/WebExpress.WebIndex.Test/IndexManagerTest.cs +++ b/src/WebExpress.WebIndex.Test/IndexManagerTest.cs @@ -3,7 +3,7 @@ internal class IndexManagerTest : WebIndex.IndexManager { /// - /// Constructor + /// Initializes a new instance of the class. /// public IndexManagerTest() { diff --git a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryA.cs b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryA.cs index 0041b25..217aaf4 100644 --- a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryA.cs +++ b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryA.cs @@ -1,5 +1,4 @@ ο»Ώusing System.Globalization; -using System.Reflection; using WebExpress.WebIndex.Memory; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Test.Fixture; @@ -12,12 +11,18 @@ namespace WebExpress.WebIndex.Test.ReverseIndex /// /// The log. /// The test context. + [Collection("NonParallelTests")] public class UnitTestReverseIndexMemoryA(UnitTestIndexFixtureIndexA fixture, ITestOutputHelper output) : UnitTestReverseIndex(fixture, output) { /// - /// Returns the property. + /// Returns the field. /// - protected static PropertyInfo Property => typeof(UnitTestIndexTestDocumentA).GetProperty("Text"); + protected static IndexFieldData Field => new() + { + Name = "Text", + PropertyInfo = typeof(UnitTestIndexTestDocumentA).GetProperty("Text"), + Type = typeof(UnitTestIndexTestDocumentA) + }; /// /// Creates a reverse index. @@ -29,7 +34,7 @@ public void Create() Preconditions(); // test execution - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); // postconditions reverseIndex.Dispose(); @@ -44,7 +49,7 @@ public void Add() { // preconditions Preconditions(); - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); @@ -69,7 +74,7 @@ public void AddToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -98,7 +103,7 @@ public void Remove() { // preconditions Preconditions(); - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -133,7 +138,7 @@ public void RemoveToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -163,7 +168,7 @@ public void Retrieve() { // preconditions Preconditions(); - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -190,7 +195,7 @@ public void All() { // preconditions Preconditions(); - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -199,9 +204,9 @@ public void All() } // test execution - //var all = reverseIndex.All; + var all = reverseIndex.All; - //Assert.True(all.Select(x => x.DocumentID).SequenceEqual(data.Select(x => x.DocumentID))); + Assert.Equal(all.OrderBy(x => x), Fixture.TestData.Select(x => x.Id).OrderBy(x => x)); // postconditions reverseIndex.Dispose(); diff --git a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryB.cs b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryB.cs index 339ef38..f3b8881 100644 --- a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryB.cs +++ b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryB.cs @@ -1,5 +1,4 @@ ο»Ώusing System.Globalization; -using System.Reflection; using WebExpress.WebIndex.Memory; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Test.Fixture; @@ -12,12 +11,28 @@ namespace WebExpress.WebIndex.Test.ReverseIndex /// /// The log. /// The test context. + [Collection("NonParallelTests")] public class UnitTestReverseIndexMemoryB(UnitTestIndexFixtureIndexB fixture, ITestOutputHelper output) : UnitTestReverseIndex(fixture, output) { /// - /// Returns the property. + /// Returns the field for the mane. /// - protected static PropertyInfo Property => typeof(UnitTestIndexTestDocumentB).GetProperty("Name"); + protected static IndexFieldData FieldName => new() + { + Name = "Name", + PropertyInfo = typeof(UnitTestIndexTestDocumentB).GetProperty("Name"), + Type = typeof(UnitTestIndexTestDocumentB) + }; + + /// + /// Returns the field for the price. + /// + protected static IndexFieldData FieldPrice => new IndexFieldData() + { + Name = "Price", + PropertyInfo = typeof(UnitTestIndexTestDocumentA).GetProperty("Price"), + Type = typeof(UnitTestIndexTestDocumentA) + }; /// /// Creates a reverse index. @@ -29,7 +44,7 @@ public void Create() Preconditions(); // test execution - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, FieldName, CultureInfo.GetCultureInfo("en")); // postconditions reverseIndex.Dispose(); @@ -44,7 +59,7 @@ public void Add() { // preconditions Preconditions(); - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, FieldName, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); @@ -69,7 +84,7 @@ public void AddToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, FieldName, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -99,7 +114,7 @@ public void Remove() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, FieldName, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -131,7 +146,7 @@ public void RemoveToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, FieldName, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -162,7 +177,7 @@ public void Retrieve() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, FieldName, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -180,6 +195,186 @@ public void Retrieve() Postconditions(); } + /// + /// Tests numeric equals. + /// + [Theory] + [InlineData(-10, 0)] + [InlineData(0, 1)] + [InlineData(10, 1)] + [InlineData(50, 1)] + [InlineData(50.5, 0)] + [InlineData(90, 1)] + [InlineData(100, 0)] + public void NumericEquals(decimal number, int expected) + { + // preconditions + Preconditions(); + var reverseIndex = new IndexMemoryReverseNumeric(Context, FieldPrice, CultureInfo.GetCultureInfo("en")); + + foreach (var item in Fixture.TestData) + { + // test execution + reverseIndex.Add(item); + } + + // test execution + var items = reverseIndex.Retrieve(number, new IndexRetrieveOptions() { Method = IndexRetrieveMethod.Phrase }); + var prices = Fixture.TestData.Where(x => items.Contains(x.Id)).Select(x => x.Price).ToList(); + + Assert.NotNull(items); + Assert.Equal(Fixture.TestData.Where(x => x.Price == (double)number).Select(x => x.Price).ToList(), prices); + Assert.Equal(expected, items.Count()); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Tests numeric greater than. + /// + [Theory] + [InlineData(-10, 100)] + [InlineData(0, 99)] + [InlineData(10, 89)] + [InlineData(50, 49)] + [InlineData(50.5, 49)] + [InlineData(90, 9)] + [InlineData(100, 0)] + public void NumericGreaterThan(decimal number, int expected) + { + // preconditions + Preconditions(); + var reverseIndex = new IndexMemoryReverseNumeric(Context, FieldPrice, CultureInfo.GetCultureInfo("en")); + + foreach (var item in Fixture.TestData) + { + // test execution + reverseIndex.Add(item); + } + + // test execution + var items = reverseIndex.Retrieve(number, new IndexRetrieveOptions() { Method = IndexRetrieveMethod.GratherThan }); + var prices = Fixture.TestData.Where(x => items.Contains(x.Id)).Select(x => x.Price).ToList(); + + Assert.NotNull(items); + Assert.Equal(Fixture.TestData.Where(x => x.Price > (double)number).Select(x => x.Price).ToList(), prices); + Assert.Equal(expected, items.Count()); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Tests numeric greater than or equals. + /// + [Theory] + [InlineData(-10, 100)] + [InlineData(0, 100)] + [InlineData(10, 90)] + [InlineData(50, 50)] + [InlineData(50.5, 49)] + [InlineData(90, 10)] + [InlineData(100, 0)] + public void NumericGreaterThanOrEquals(decimal number, int expected) + { + // preconditions + Preconditions(); + var reverseIndex = new IndexMemoryReverseNumeric(Context, FieldPrice, CultureInfo.GetCultureInfo("en")); + + foreach (var item in Fixture.TestData) + { + // test execution + reverseIndex.Add(item); + } + + // test execution + var items = reverseIndex.Retrieve(number, new IndexRetrieveOptions() { Method = IndexRetrieveMethod.GratherThanOrEqual }); + var prices = Fixture.TestData.Where(x => items.Contains(x.Id)).Select(x => x.Price).ToList(); + + Assert.NotNull(items); + Assert.Equal(Fixture.TestData.Where(x => x.Price >= (double)number).Select(x => x.Price).ToList(), prices); + Assert.Equal(expected, items.Count()); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Tests numeric less than or equals. + /// + [Theory] + [InlineData(-10, 0)] + [InlineData(0, 0)] + [InlineData(10, 10)] + [InlineData(50, 50)] + [InlineData(50.5, 51)] + [InlineData(90, 90)] + [InlineData(100, 100)] + public void NumericLessThan(decimal number, int expected) + { + // preconditions + Preconditions(); + var reverseIndex = new IndexMemoryReverseNumeric(Context, FieldPrice, CultureInfo.GetCultureInfo("en")); + + foreach (var item in Fixture.TestData) + { + // test execution + reverseIndex.Add(item); + } + + // test execution + var items = reverseIndex.Retrieve(number, new IndexRetrieveOptions() { Method = IndexRetrieveMethod.LessThan }); + var prices = Fixture.TestData.Where(x => items.Contains(x.Id)).Select(x => x.Price).ToList(); + + Assert.NotNull(items); + Assert.Equal(Fixture.TestData.Where(x => x.Price < (double)number).Select(x => x.Price).ToList(), prices); + Assert.Equal(expected, items.Count()); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Tests numeric less than or equals. + /// + [Theory] + [InlineData(-10, 0)] + [InlineData(0, 1)] + [InlineData(10, 11)] + [InlineData(50, 51)] + [InlineData(50.5, 51)] + [InlineData(90, 91)] + [InlineData(100, 100)] + public void NumericLessThanOrEquals(decimal number, int expected) + { + // preconditions + Preconditions(); + var reverseIndex = new IndexMemoryReverseNumeric(Context, FieldPrice, CultureInfo.GetCultureInfo("en")); + + foreach (var item in Fixture.TestData) + { + // test execution + reverseIndex.Add(item); + } + + // test execution + var items = reverseIndex.Retrieve(number, new IndexRetrieveOptions() { Method = IndexRetrieveMethod.LessThanOrEqual }); + var prices = Fixture.TestData.Where(x => items.Contains(x.Id)).Select(x => x.Price).ToList(); + + Assert.NotNull(items); + Assert.Equal(Fixture.TestData.Where(x => x.Price <= (double)number).Select(x => x.Price).ToList(), prices); + Assert.Equal(expected, items.Count()); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + /// /// Return all entries of the reverse index. /// @@ -188,7 +383,7 @@ public void All() { // preconditions Preconditions(); - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, FieldName, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -198,9 +393,9 @@ public void All() } // test execution - //var all = reverseIndex.All; + var all = reverseIndex.All; - //Assert.True(all.Select(x => x.DocumentID).SequenceEqual(data.Select(x => x.DocumentID))); + Assert.Equal(all.OrderBy(x => x), Fixture.TestData.Select(x => x.Id).OrderBy(x => x)); // postconditions reverseIndex.Dispose(); @@ -208,3 +403,4 @@ public void All() } } } + diff --git a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryC.cs b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryC.cs index dfff2df..5aa1e3e 100644 --- a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryC.cs +++ b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryC.cs @@ -1,5 +1,4 @@ ο»Ώusing System.Globalization; -using System.Reflection; using WebExpress.WebIndex.Memory; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Test.Fixture; @@ -12,12 +11,18 @@ namespace WebExpress.WebIndex.Test.ReverseIndex /// /// The log. /// The test context. + [Collection("NonParallelTests")] public class UnitTestReverseIndexMemoryC(UnitTestIndexFixtureIndexC fixture, ITestOutputHelper output) : UnitTestReverseIndex(fixture, output) { /// - /// Returns the property. + /// Returns the field. /// - protected static PropertyInfo Property => typeof(UnitTestIndexTestDocumentC).GetProperty("Text"); + protected static IndexFieldData Field => new() + { + Name = "Text", + PropertyInfo = typeof(UnitTestIndexTestDocumentC).GetProperty("Text"), + Type = typeof(UnitTestIndexTestDocumentC) + }; /// /// Creates a reverse index. @@ -29,7 +34,7 @@ public void Create() Preconditions(); // test execution - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); // postconditions reverseIndex.Dispose(); @@ -44,7 +49,7 @@ public void Add() { // preconditions Preconditions(); - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); @@ -70,7 +75,7 @@ public void AddToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -100,7 +105,7 @@ public void Remove() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -132,7 +137,7 @@ public void RemoveToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -163,7 +168,7 @@ public void Retrieve() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -189,7 +194,7 @@ public void All() { // preconditions Preconditions(); - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) diff --git a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryD.cs b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryD.cs index eed2d4d..8de6b46 100644 --- a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryD.cs +++ b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryD.cs @@ -1,5 +1,4 @@ ο»Ώusing System.Globalization; -using System.Reflection; using WebExpress.WebIndex.Memory; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Test.Fixture; @@ -12,12 +11,18 @@ namespace WebExpress.WebIndex.Test.ReverseIndex /// /// The log. /// The test context. + [Collection("NonParallelTests")] public class UnitTestReverseIndexMemoryD(UnitTestIndexFixtureIndexD fixture, ITestOutputHelper output) : UnitTestReverseIndex(fixture, output) { /// - /// Returns the property. + /// Returns the field. /// - protected static PropertyInfo Property => typeof(UnitTestIndexTestDocumentD).GetProperty("FirstName"); + protected static IndexFieldData Field => new() + { + Name = "FirstName", + PropertyInfo = typeof(UnitTestIndexTestDocumentD).GetProperty("FirstName"), + Type = typeof(UnitTestIndexTestDocumentD) + }; /// /// Creates a reverse index. @@ -29,7 +34,7 @@ public void Create() Preconditions(); // test execution - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); // postconditions reverseIndex.Dispose(); @@ -44,7 +49,7 @@ public void Add() { // preconditions Preconditions(); - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); @@ -69,7 +74,7 @@ public void AddToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -99,7 +104,7 @@ public void Remove() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -131,7 +136,7 @@ public void RemoveToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -162,7 +167,7 @@ public void Retrieve() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -188,7 +193,7 @@ public void All() { // preconditions Preconditions(); - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -198,9 +203,9 @@ public void All() } // test execution - //var all = reverseIndex.All; + var all = reverseIndex.All; - //Assert.True(all.Select(x => x.DocumentID).SequenceEqual(data.Select(x => x.DocumentID))); + Assert.Equal(all.OrderBy(x => x), Fixture.TestData.Select(x => x.Id).OrderBy(x => x)); // postconditions reverseIndex.Dispose(); diff --git a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryE.cs b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryE.cs index fd9d368..3facef2 100644 --- a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryE.cs +++ b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryE.cs @@ -1,5 +1,4 @@ ο»Ώusing System.Globalization; -using System.Reflection; using WebExpress.WebIndex.Memory; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Test.Fixture; @@ -12,12 +11,18 @@ namespace WebExpress.WebIndex.Test.ReverseIndex /// /// The log. /// The test context. + [Collection("NonParallelTests")] public class UnitTestReverseIndexMemoryE(UnitTestIndexFixtureIndexE fixture, ITestOutputHelper output) : UnitTestReverseIndex(fixture, output) { /// - /// Returns the property. + /// Returns the field. /// - protected static PropertyInfo Property => typeof(UnitTestIndexTestDocumentE).GetProperty("Name"); + protected static IndexFieldData Field => new() + { + Name = "Name", + PropertyInfo = typeof(UnitTestIndexTestDocumentE).GetProperty("Name"), + Type = typeof(UnitTestIndexTestDocumentE) + }; /// /// Creates a reverse index. @@ -29,7 +34,7 @@ public void Create() Preconditions(); // test execution - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); // postconditions reverseIndex.Dispose(); @@ -44,7 +49,7 @@ public void Add() { // preconditions Preconditions(); - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); @@ -69,7 +74,7 @@ public void AddToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -99,7 +104,7 @@ public void Remove() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -131,7 +136,7 @@ public void RemoveToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -162,7 +167,7 @@ public void Retrieve() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -188,7 +193,7 @@ public void All() { // preconditions Preconditions(); - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -198,9 +203,9 @@ public void All() } // test execution - //var all = reverseIndex.All; + var all = reverseIndex.All; - //Assert.True(all.Select(x => x.DocumentID).SequenceEqual(data.Select(x => x.DocumentID))); + Assert.Equal(all.OrderBy(x => x), Fixture.TestData.Select(x => x.Id).OrderBy(x => x)); // postconditions reverseIndex.Dispose(); diff --git a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryF.cs b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryF.cs new file mode 100644 index 0000000..4bb1b75 --- /dev/null +++ b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexMemoryF.cs @@ -0,0 +1,194 @@ +ο»Ώusing System.Globalization; +using WebExpress.WebIndex.Memory; +using WebExpress.WebIndex.Test.Document; +using WebExpress.WebIndex.Test.Fixture; +using Xunit.Abstractions; + +namespace WebExpress.WebIndex.Test.ReverseIndex +{ + /// + /// Test class for testing the memory-based reverse index for unicode. + /// + /// The log. + /// The test context. + [Collection("NonParallelTests")] + public class UnitTestReverseIndexMemoryF(UnitTestIndexFixtureIndexF fixture, ITestOutputHelper output) : UnitTestReverseIndex(fixture, output) + { + /// + /// Returns the field. + /// + protected static IndexFieldData Field => new() + { + Name = "Name", + PropertyInfo = typeof(UnitTestIndexTestDocumentF).GetProperty("Name"), + Type = typeof(UnitTestIndexTestDocumentF) + }; + + /// + /// Creates a reverse index. + /// + [Fact] + public void Create() + { + // preconditions + Preconditions(); + + // test execution + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Adds items to a reverse index. + /// + [Fact] + public void Add() + { + // preconditions + Preconditions(); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); + + reverseIndex.Clear(); + + foreach (var item in Fixture.TestData) + { + // test execution + reverseIndex.Add(item); + } + + Assert.NotNull(reverseIndex); + + // postconditions + Postconditions(); + } + + /// + /// Adds a token to an existing entry in the reverse index. + /// + [Theory] + [InlineData("Aurora", true)] + [InlineData("😊🌸🐼", false)] + [InlineData("张伟", true)] + public void AddToken(string str, bool valid) + { + // preconditions + Preconditions(); + var doc = new UnitTestIndexTestDocumentF() { Id = Guid.Parse("9A274C29-E210-49C9-A673-238F79636CD9"), Name = str }; + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); + var token = Context.TokenAnalyzer.Analyze(str, CultureInfo.GetCultureInfo("en")); + + // test execution + reverseIndex.Add(doc, token); + var item = reverseIndex.Retrieve(str, new IndexRetrieveOptions()); + + if (valid) + { + Assert.Contains(doc.Id, item); + } + else + { + Assert.Empty(item); + } + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Removes a token without deleting the entire entry. + /// + [Fact] + public void Remove() + { + // preconditions + Preconditions(); + var randomItem = Fixture.RandomItem; + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); + + reverseIndex.Clear(); + foreach (var item in Fixture.TestData) + { + reverseIndex.Add(item); + } + + var token = Context.TokenAnalyzer.Analyze("Aurora", CultureInfo.GetCultureInfo("en")); + reverseIndex.Add(randomItem, token.TakeLast(1)); + + // test execution + reverseIndex.Delete(randomItem, token.TakeLast(1)); + + var items = reverseIndex.Retrieve("aurora", new IndexRetrieveOptions()); + Assert.Empty(items); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Retrieve a entry of the reverse index. + /// + [Fact] + public void Retrieve() + { + // preconditions + Preconditions(); + var randomItem = Fixture.RandomItem; + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); + + reverseIndex.Clear(); + foreach (var item in Fixture.TestData) + { + // test execution + reverseIndex.Add(item); + } + + // test execution + var items = reverseIndex.Retrieve(randomItem.Name, new IndexRetrieveOptions()); + + if (randomItem.Name != "😊🌸🐼") + { + Assert.True(items.Any()); + } + else + { + Assert.Empty(items); + } + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Return all entries of the reverse index. + /// + [Fact] + public void All() + { + // preconditions + Preconditions(); + var reverseIndex = new IndexMemoryReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); + + reverseIndex.Clear(); + foreach (var item in Fixture.TestData) + { + // test execution + reverseIndex.Add(item); + } + + // test execution + var all = reverseIndex.All; + + Assert.Equal(all.OrderBy(x => x), Fixture.TestData.Where(x => x.Name != "😊🌸🐼").Select(x => x.Id).OrderBy(x => x)); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + } +} diff --git a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageA.cs b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageA.cs index fdefe6b..c814075 100644 --- a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageA.cs +++ b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageA.cs @@ -1,6 +1,4 @@ ο»Ώusing System.Globalization; -using System.Reflection; -using WebExpress.WebIndex.Memory; using WebExpress.WebIndex.Storage; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Test.Fixture; @@ -13,12 +11,18 @@ namespace WebExpress.WebIndex.Test.ReverseIndex /// /// The log. /// The test context. + [Collection("NonParallelTests")] public class UnitTestReverseIndexStorageA(UnitTestIndexFixtureIndexA fixture, ITestOutputHelper output) : UnitTestReverseIndex(fixture, output) { /// - /// Returns the property. + /// Returns the field. /// - protected static PropertyInfo Property => typeof(UnitTestIndexTestDocumentA).GetProperty("Text"); + protected static IndexFieldData Field => new() + { + Name = "Text", + PropertyInfo = typeof(UnitTestIndexTestDocumentA).GetProperty("Text"), + Type = typeof(UnitTestIndexTestDocumentA) + }; /// /// Creates a reverse index. @@ -30,7 +34,7 @@ public void Create() Preconditions(); // test execution - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); // postconditions reverseIndex.Dispose(); @@ -45,7 +49,7 @@ public void Add() { // preconditions Preconditions(); - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); @@ -62,6 +66,36 @@ public void Add() Postconditions(); } + /// + /// Adds items with surrogate character to a reverse index. + /// + [Fact] + public void AddSurrogate() + { + // preconditions + Preconditions(); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); + + reverseIndex.Clear(); + + var chars = new char[] { '\uD800', '\uDC00' }; // this is a surrogate pair + + var item = new UnitTestIndexTestDocumentA() + { + Id = Guid.NewGuid(), + Text = $"abc{new string(chars)}def" + }; + + // test execution + reverseIndex.Add(item); + + Assert.Empty(reverseIndex.All); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + /// /// Adds a token to an existing entry in the reverse index. /// @@ -71,7 +105,7 @@ public void AddToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -101,7 +135,7 @@ public void Remove() { // preconditions Preconditions(); - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -135,7 +169,7 @@ public void RemoveToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -169,7 +203,7 @@ public void Retrieve() { // preconditions Preconditions(); - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -195,7 +229,7 @@ public void All() { // preconditions Preconditions(); - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) diff --git a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageB.cs b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageB.cs index 24bfe9b..c7321f5 100644 --- a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageB.cs +++ b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageB.cs @@ -1,6 +1,5 @@ ο»Ώusing System.Globalization; -using System.Reflection; -using WebExpress.WebIndex.Memory; +using WebExpress.WebIndex.Storage; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Test.Fixture; using Xunit.Abstractions; @@ -12,12 +11,28 @@ namespace WebExpress.WebIndex.Test.ReverseIndex /// /// The log. /// The test context. + [Collection("NonParallelTests")] public class UnitTestReverseIndexStorageB(UnitTestIndexFixtureIndexB fixture, ITestOutputHelper output) : UnitTestReverseIndex(fixture, output) { /// - /// Test class for testing the storage-based reverse index. + /// Returns the property for the name. /// - protected static PropertyInfo Property => typeof(UnitTestIndexTestDocumentB).GetProperty("Name"); + protected static IndexFieldData FieldName => new() + { + Name = "Name", + PropertyInfo = typeof(UnitTestReverseIndexStorageB).GetProperty("Name"), + Type = typeof(UnitTestReverseIndexStorageB) + }; + + /// + /// Returns the property for the price. + /// + protected static IndexFieldData FieldPrice => new() + { + Name = "Price", + PropertyInfo = typeof(UnitTestReverseIndexStorageB).GetProperty("Price"), + Type = typeof(UnitTestReverseIndexStorageB) + }; /// /// Creates a reverse index. @@ -29,7 +44,7 @@ public void Create() Preconditions(); // test execution - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, FieldName, CultureInfo.GetCultureInfo("en")); // postconditions reverseIndex.Dispose(); @@ -44,9 +59,7 @@ public void Add() { // preconditions Preconditions(); - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); - - reverseIndex.Clear(); + var reverseIndex = new IndexStorageReverseTerm(Context, FieldName, CultureInfo.GetCultureInfo("en")); foreach (var item in Fixture.TestData) { @@ -54,9 +67,10 @@ public void Add() reverseIndex.Add(item); } - Assert.NotNull(reverseIndex); + Assert.NotEmpty(reverseIndex.All); // postconditions + reverseIndex.Dispose(); Postconditions(); } @@ -69,7 +83,7 @@ public void AddToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, FieldName, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -99,7 +113,7 @@ public void Remove() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, FieldName, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -131,7 +145,7 @@ public void RemoveToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, FieldName, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -162,7 +176,7 @@ public void Retrieve() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, FieldName, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -180,6 +194,186 @@ public void Retrieve() Postconditions(); } + /// + /// Tests numeric equals. + /// + [Theory] + [InlineData(-10, 0)] + [InlineData(0, 1)] + [InlineData(10, 1)] + [InlineData(50, 1)] + [InlineData(50.5, 0)] + [InlineData(90, 1)] + [InlineData(100, 0)] + public void NumericEquals(decimal number, int expected) + { + // preconditions + Preconditions(); + var reverseIndex = new IndexStorageReverseNumeric(Context, FieldPrice, CultureInfo.GetCultureInfo("en")); + + foreach (var item in Fixture.TestData) + { + // test execution + reverseIndex.Add(item); + } + + // test execution + var items = reverseIndex.Retrieve(number, new IndexRetrieveOptions() { Method = IndexRetrieveMethod.Phrase }); + var prices = Fixture.TestData.Where(x => items.Contains(x.Id)).Select(x => x.Price).ToList(); + + Assert.NotNull(items); + Assert.Equal(Fixture.TestData.Where(x => x.Price == (double)number).Select(x => x.Price).ToList(), prices); + Assert.Equal(expected, items.Count()); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Tests numeric greater than. + /// + [Theory] + [InlineData(-10, 100)] + [InlineData(0, 99)] + [InlineData(10, 89)] + [InlineData(50, 49)] + [InlineData(50.5, 49)] + [InlineData(90, 9)] + [InlineData(100, 0)] + public void NumericGreaterThan(decimal number, int expected) + { + // preconditions + Preconditions(); + var reverseIndex = new IndexStorageReverseNumeric(Context, FieldPrice, CultureInfo.GetCultureInfo("en")); + + foreach (var item in Fixture.TestData) + { + // test execution + reverseIndex.Add(item); + } + + // test execution + var items = reverseIndex.Retrieve(number, new IndexRetrieveOptions() { Method = IndexRetrieveMethod.GratherThan }); + var prices = Fixture.TestData.Where(x => items.Contains(x.Id)).Select(x => x.Price).ToList(); + + Assert.NotNull(items); + Assert.Equal(Fixture.TestData.Where(x => x.Price > (double)number).Select(x => x.Price).ToList(), prices); + Assert.Equal(expected, items.Count()); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Tests numeric greater than or equals. + /// + [Theory] + [InlineData(-10, 100)] + [InlineData(0, 100)] + [InlineData(10, 90)] + [InlineData(50, 50)] + [InlineData(50.5, 49)] + [InlineData(90, 10)] + [InlineData(100, 0)] + public void NumericGreaterThanOrEquals(decimal number, int expected) + { + // preconditions + Preconditions(); + var reverseIndex = new IndexStorageReverseNumeric(Context, FieldPrice, CultureInfo.GetCultureInfo("en")); + + foreach (var item in Fixture.TestData) + { + // test execution + reverseIndex.Add(item); + } + + // test execution + var items = reverseIndex.Retrieve(number, new IndexRetrieveOptions() { Method = IndexRetrieveMethod.GratherThanOrEqual }); + var prices = Fixture.TestData.Where(x => items.Contains(x.Id)).Select(x => x.Price).ToList(); + + Assert.NotNull(items); + Assert.Equal(Fixture.TestData.Where(x => x.Price >= (double)number).Select(x => x.Price).ToList(), prices); + Assert.Equal(expected, items.Count()); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Tests numeric less than or equals. + /// + [Theory] + [InlineData(-10, 0)] + [InlineData(0, 0)] + [InlineData(10, 10)] + [InlineData(50, 50)] + [InlineData(50.5, 51)] + [InlineData(90, 90)] + [InlineData(100, 100)] + public void NumericLessThan(decimal number, int expected) + { + // preconditions + Preconditions(); + var reverseIndex = new IndexStorageReverseNumeric(Context, FieldPrice, CultureInfo.GetCultureInfo("en")); + + foreach (var item in Fixture.TestData) + { + // test execution + reverseIndex.Add(item); + } + + // test execution + var items = reverseIndex.Retrieve(number, new IndexRetrieveOptions() { Method = IndexRetrieveMethod.LessThan }); + var prices = Fixture.TestData.Where(x => items.Contains(x.Id)).Select(x => x.Price).ToList(); + + Assert.NotNull(items); + Assert.Equal(Fixture.TestData.Where(x => x.Price < (double)number).Select(x => x.Price).ToList(), prices); + Assert.Equal(expected, items.Count()); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Tests numeric less than or equals. + /// + [Theory] + [InlineData(-10, 0)] + [InlineData(0, 1)] + [InlineData(10, 11)] + [InlineData(50, 51)] + [InlineData(50.5, 51)] + [InlineData(90, 91)] + [InlineData(100, 100)] + public void NumericLessThanOrEquals(decimal number, int expected) + { + // preconditions + Preconditions(); + var reverseIndex = new IndexStorageReverseNumeric(Context, FieldPrice, CultureInfo.GetCultureInfo("en")); + + foreach (var item in Fixture.TestData) + { + // test execution + reverseIndex.Add(item); + } + + // test execution + var items = reverseIndex.Retrieve(number, new IndexRetrieveOptions() { Method = IndexRetrieveMethod.LessThanOrEqual }); + var prices = Fixture.TestData.Where(x => items.Contains(x.Id)).Select(x => x.Price).ToList(); + + Assert.NotNull(items); + Assert.Equal(Fixture.TestData.Where(x => x.Price <= (double)number).Select(x => x.Price).ToList(), prices); + Assert.Equal(expected, items.Count()); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + /// /// Return all entries of the reverse index. /// @@ -188,7 +382,7 @@ public void All() { // preconditions Preconditions(); - var reverseIndex = new IndexMemoryReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, FieldName, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -198,9 +392,9 @@ public void All() } // test execution - //var all = reverseIndex.All; + var all = reverseIndex.All.OrderBy(x => x).ToList(); - //Assert.True(all.Select(x => x.DocumentID).SequenceEqual(data.Select(x => x.DocumentID))); + Assert.Equal(all, Fixture.TestData.Select(x => x.Id).OrderBy(x => x).ToList()); // postconditions reverseIndex.Dispose(); diff --git a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageC.cs b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageC.cs index 885dea3..1c8f3bf 100644 --- a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageC.cs +++ b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageC.cs @@ -1,5 +1,4 @@ ο»Ώusing System.Globalization; -using System.Reflection; using WebExpress.WebIndex.Storage; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Test.Fixture; @@ -12,12 +11,18 @@ namespace WebExpress.WebIndex.Test.ReverseIndex /// /// The log. /// The test context. + [Collection("NonParallelTests")] public class UnitTestReverseIndexStorageC(UnitTestIndexFixtureIndexC fixture, ITestOutputHelper output) : UnitTestReverseIndex(fixture, output) { /// - /// Test class for testing the storage-based reverse index. + /// Returns the field. /// - protected static PropertyInfo Property => typeof(UnitTestIndexTestDocumentC).GetProperty("Text"); + protected static IndexFieldData Field => new() + { + Name = "Text", + PropertyInfo = typeof(UnitTestIndexTestDocumentC).GetProperty("Text"), + Type = typeof(UnitTestIndexTestDocumentC) + }; /// /// Creates a reverse index. @@ -29,7 +34,7 @@ public void Create() Preconditions(); // test execution - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); // postconditions reverseIndex.Dispose(); @@ -44,7 +49,7 @@ public void Add() { // preconditions Preconditions(); - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); @@ -70,7 +75,7 @@ public void AddToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -100,7 +105,7 @@ public void Remove() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -132,7 +137,7 @@ public void RemoveToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -163,7 +168,7 @@ public void Retrieve() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -189,7 +194,7 @@ public void All() { // preconditions Preconditions(); - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) diff --git a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageD.cs b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageD.cs index e4bf277..8bf3339 100644 --- a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageD.cs +++ b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageD.cs @@ -1,5 +1,4 @@ ο»Ώusing System.Globalization; -using System.Reflection; using WebExpress.WebIndex.Storage; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Test.Fixture; @@ -12,12 +11,18 @@ namespace WebExpress.WebIndex.Test.ReverseIndex /// /// The log. /// The test context. + [Collection("NonParallelTests")] public class UnitTestReverseIndexStorageD(UnitTestIndexFixtureIndexD fixture, ITestOutputHelper output) : UnitTestReverseIndex(fixture, output) { /// - /// Test class for testing the storage-based reverse index. + /// Returns the field. /// - protected static PropertyInfo Property => typeof(UnitTestIndexTestDocumentD).GetProperty("FirstName"); + protected static IndexFieldData Field => new() + { + Name = "FirstName", + PropertyInfo = typeof(UnitTestIndexTestDocumentD).GetProperty("FirstName"), + Type = typeof(UnitTestIndexTestDocumentD) + }; /// /// Creates a reverse index. @@ -29,7 +34,7 @@ public void Create() Preconditions(); // test execution - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); // postconditions reverseIndex.Dispose(); @@ -44,7 +49,7 @@ public void Add() { // preconditions Preconditions(); - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); @@ -70,7 +75,7 @@ public void AddToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -100,7 +105,7 @@ public void Remove() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -132,7 +137,7 @@ public void RemoveToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -163,7 +168,7 @@ public void Retrieve() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -189,7 +194,7 @@ public void All() { // preconditions Preconditions(); - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) diff --git a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageE.cs b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageE.cs index 06b82f1..da16b42 100644 --- a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageE.cs +++ b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageE.cs @@ -1,5 +1,4 @@ ο»Ώusing System.Globalization; -using System.Reflection; using WebExpress.WebIndex.Storage; using WebExpress.WebIndex.Test.Document; using WebExpress.WebIndex.Test.Fixture; @@ -12,12 +11,18 @@ namespace WebExpress.WebIndex.Test.ReverseIndex /// /// The log. /// The test context. + [Collection("NonParallelTests")] public class UnitTestReverseIndexStorageE(UnitTestIndexFixtureIndexE fixture, ITestOutputHelper output) : UnitTestReverseIndex(fixture, output) { /// - /// Test class for testing the storage-based reverse index. + /// Returns the field. /// - protected static PropertyInfo Property => typeof(UnitTestIndexTestDocumentE).GetProperty("Name"); + protected static IndexFieldData Field => new() + { + Name = "Name", + PropertyInfo = typeof(UnitTestIndexTestDocumentE).GetProperty("Name"), + Type = typeof(UnitTestIndexTestDocumentE) + }; /// /// Creates a reverse index. @@ -29,7 +34,7 @@ public void Create() Preconditions(); // test execution - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); // postconditions reverseIndex.Dispose(); @@ -44,7 +49,7 @@ public void Add() { // preconditions Preconditions(); - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); @@ -70,7 +75,7 @@ public void AddToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -100,7 +105,7 @@ public void Remove() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -132,7 +137,7 @@ public void RemoveToken() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -163,7 +168,7 @@ public void Retrieve() // preconditions Preconditions(); var randomItem = Fixture.RandomItem; - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) @@ -189,7 +194,7 @@ public void All() { // preconditions Preconditions(); - var reverseIndex = new IndexStorageReverse(Context, Property, CultureInfo.GetCultureInfo("en")); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); reverseIndex.Clear(); foreach (var item in Fixture.TestData) diff --git a/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageF.cs b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageF.cs new file mode 100644 index 0000000..8557f4c --- /dev/null +++ b/src/WebExpress.WebIndex.Test/ReverseIndex/UnitTestReverseIndexStorageF.cs @@ -0,0 +1,141 @@ +ο»Ώusing System.Globalization; +using WebExpress.WebIndex.Storage; +using WebExpress.WebIndex.Test.Document; +using WebExpress.WebIndex.Test.Fixture; +using Xunit.Abstractions; + +namespace WebExpress.WebIndex.Test.ReverseIndex +{ + /// + /// Test class for testing the storage-based reverse index for unicode. + /// + /// The log. + /// The test context. + [Collection("NonParallelTests")] + public class UnitTestReverseIndexStorageF(UnitTestIndexFixtureIndexF fixture, ITestOutputHelper output) : UnitTestReverseIndex(fixture, output) + { + /// + /// Returns the field. + /// + protected static IndexFieldData Field => new() + { + Name = "Name", + PropertyInfo = typeof(UnitTestIndexTestDocumentF).GetProperty("Name"), + Type = typeof(UnitTestIndexTestDocumentF) + }; + + /// + /// Creates a reverse index. + /// + [Fact] + public void Create() + { + // preconditions + Preconditions(); + + // test execution + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Adds items to a reverse index. + /// + [Theory] + [InlineData("ED242C79-E41B-4214-BFBC-C4673E87433B", "Aurora", true)] + [InlineData("A20BC371-10F9-4F43-9DA8-F4B4F0BE26AB", "李明", true)] + [InlineData("9733A649-1E5E-4B1F-8C6E-9A4B6AB54292", "πŸŒŸπŸ€πŸ‰", false)] + [InlineData("80A78EBB-9819-45AF-BC0F-68E68D0C8C1A", "Sun Leaf Lion 🌞🌿🦁", true)] + [InlineData("29F34DFD-432D-4315-88C2-CE41F293AC71", "πŸ¦‹πŸŒΌπŸŒ™ Butterfly Flower Moon", true)] + public void Add(string id, string name, bool valid) + { + // preconditions + Preconditions(); + var doc = new UnitTestIndexTestDocumentF() { Id = Guid.Parse(id), Name = name }; + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); + + // test execution + reverseIndex.Add(new UnitTestIndexTestDocumentF() { Id = Guid.Parse(id), Name = name }); + + Assert.NotNull(reverseIndex); + + var items = reverseIndex.Retrieve(name, new IndexRetrieveOptions()); + + if (valid) + { + Assert.Contains(doc.Id, items); + } + else + { + Assert.Empty(items); + } + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Removes a token without deleting the entire entry. + /// + [Theory] + [InlineData("Aurora")] + [InlineData("张伟")] + public void Remove(string str) + { + // preconditions + Preconditions(); + var randomItem = Fixture.TestData?.LastOrDefault(); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); + + foreach (var item in Fixture.TestData) + { + reverseIndex.Add(item); + } + + var token = Context.TokenAnalyzer.Analyze(str, CultureInfo.GetCultureInfo("en")); + reverseIndex.Add(randomItem, token.TakeLast(1)); + + // test execution + reverseIndex.Delete(randomItem, token.TakeLast(1)); + + var items = reverseIndex.Retrieve(str, new IndexRetrieveOptions()); + Assert.Empty(items); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + + /// + /// Retrieve a entry of the reverse index. + /// + [Theory] + [InlineData("张伟", IndexRetrieveMethod.Phrase)] + public void Retrieve(string str, IndexRetrieveMethod method) + { + // preconditions + Preconditions(); + var reverseIndex = new IndexStorageReverseTerm(Context, Field, CultureInfo.GetCultureInfo("en")); + var option = new IndexRetrieveOptions() { Method = method }; + + foreach (var item in Fixture.TestData) + { + // test execution + reverseIndex.Add(item); + } + + // test execution + var items = reverseIndex.Retrieve(str, option); + + Assert.NotEmpty(items); + + // postconditions + reverseIndex.Dispose(); + Postconditions(); + } + } +} diff --git a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocument.cs b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocument.cs index ff8236e..4593438 100644 --- a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocument.cs +++ b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocument.cs @@ -1,8 +1,10 @@ -ο»Ώusing System.Text; -using WebExpress.WebIndex.WebAttribute; +ο»Ώusing WebExpress.WebIndex.WebAttribute; namespace WebExpress.WebIndex.Test.Document { + /// + /// Abstract class representing a unit test document for index testing. + /// public abstract class UnitTestIndexTestDocument : IIndexItem { /// diff --git a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentB.cs b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentB.cs index 90cc634..df0a27c 100644 --- a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentB.cs +++ b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentB.cs @@ -1,16 +1,54 @@ -ο»Ώnamespace WebExpress.WebIndex.Test.Document -{ +ο»Ώusing WebExpress.WebIndex.WebAttribute; +namespace WebExpress.WebIndex.Test.Document +{ /// /// Data class for unit testing. /// public class UnitTestIndexTestDocumentB : UnitTestIndexTestDocument { + /// + /// Enumeration representing various countries. + /// + public enum Country + { + Germany, + France, + Italy, + Spain, + England, + USA, + Canada, + Australia, + Japan, + China + } + + /// + /// Class representing an address with street, city, and zip code. + /// public class AdressClass { - public string Street { get; set;} - public string City { get; set;} - public int Zip { get; set;} + /// + /// Returns or sets the country. + /// + public Country Country { get; set; } + + /// + /// Returns or sets the street. + /// + public string Street { get; set; } + + /// + /// Returns or sets the city. + /// + public string City { get; set; } + + /// + /// Returns or sets the zip code. + /// + [IndexIgnore] + public int Zip { get; set; } } /// diff --git a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentF.cs b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentF.cs new file mode 100644 index 0000000..6d36844 --- /dev/null +++ b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentF.cs @@ -0,0 +1,24 @@ +ο»Ώnamespace WebExpress.WebIndex.Test.Document +{ + /// + /// Represents a test document for unicode. + /// + public class UnitTestIndexTestDocumentF : UnitTestIndexTestDocument + { + /// + /// Returns or sets the name. + /// + public string Name { get; set; } + + /// + /// Convert the object into a string representation. + /// + /// + /// A string that represents the current object. + /// + public override string ToString() + { + return $"{Id} - {Name}"; + } + } +} diff --git a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactory.cs b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactory.cs index e8ad0ab..52a288f 100644 --- a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactory.cs +++ b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactory.cs @@ -2,6 +2,9 @@ namespace WebExpress.WebIndex.Test.Document { + /// + /// Abstract factory class for creating unit test documents for indexing. + /// public abstract class UnitTestIndexTestDocumentFactory { /// diff --git a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryA.cs b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryA.cs index 5042b41..44a1bc3 100644 --- a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryA.cs +++ b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryA.cs @@ -1,7 +1,7 @@ ο»Ώnamespace WebExpress.WebIndex.Test.Document { /// - /// Represents a test document for a person. + /// Factory class for creating unit test documents of type UnitTestIndexTestDocumentA. /// public class UnitTestIndexTestDocumentFactoryA : UnitTestIndexTestDocumentFactory { diff --git a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryB.cs b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryB.cs index 18283c6..bef5647 100644 --- a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryB.cs +++ b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryB.cs @@ -1,5 +1,10 @@ -ο»Ώnamespace WebExpress.WebIndex.Test.Document +ο»Ώusing static WebExpress.WebIndex.Test.Document.UnitTestIndexTestDocumentB; + +namespace WebExpress.WebIndex.Test.Document { + /// + /// Factory class for creating unit test documents of type UnitTestIndexTestDocumentB. + /// public class UnitTestIndexTestDocumentFactoryB : UnitTestIndexTestDocumentFactory { /// @@ -19,12 +24,18 @@ public static List GenerateTestData() { Id = Guid.NewGuid(), Name = $"Name_{i}", - Summary = $"Der Name_{i}", + Summary = $"The Name_{i}", Description = GenerateLoremIpsum(100), Date = DateTime.Now.AddMonths(i % 12), Price = i, New = i % 2 != 0, - Adress = new UnitTestIndexTestDocumentB.AdressClass() { City = GenerateCity(1), Zip = GenerateZip(i), Street = GenerateSreet(i)} + Adress = new UnitTestIndexTestDocumentB.AdressClass() + { + Country = Country.USA, + City = GenerateCity(1), + Zip = GenerateZip(i), + Street = GenerateSreet(i) + } }); } diff --git a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryC.cs b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryC.cs index fbcc66d..5063c84 100644 --- a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryC.cs +++ b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryC.cs @@ -1,5 +1,8 @@ ο»Ώnamespace WebExpress.WebIndex.Test.Document { + /// + /// Factory class for creating unit test documents of type UnitTestIndexTestDocumentC. + /// public class UnitTestIndexTestDocumentFactoryC : UnitTestIndexTestDocumentFactory { /// diff --git a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryD.cs b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryD.cs index d2739a5..06cb83d 100644 --- a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryD.cs +++ b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryD.cs @@ -1,5 +1,8 @@ ο»Ώnamespace WebExpress.WebIndex.Test.Document { + /// + /// Factory class for creating unit test documents of type UnitTestIndexTestDocumentD. + /// public class UnitTestIndexTestDocumentFactoryD : UnitTestIndexTestDocumentFactory { /// diff --git a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryE.cs b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryE.cs index c8a1e42..334eb46 100644 --- a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryE.cs +++ b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryE.cs @@ -1,7 +1,7 @@ ο»Ώnamespace WebExpress.WebIndex.Test.Document { /// - /// Represents a test document for a person. + /// Factory class for creating unit test documents of type UnitTestIndexTestDocumentE. /// public class UnitTestIndexTestDocumentFactoryE : UnitTestIndexTestDocumentFactory { @@ -139,9 +139,13 @@ public static List GenerateTestData() { Id = Guid.Parse("160a6cba-5f74-4d0d-939a-5e0764276b04"), Name = "Lars" + }, + new () + { + Id = Guid.Parse("4fa7861e-54cd-408c-8651-0dd05cb94bd1"), + Name = "RenΓ©" } - }; return testDataList; diff --git a/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryF.cs b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryF.cs new file mode 100644 index 0000000..797c68a --- /dev/null +++ b/src/WebExpress.WebIndex.Test/TestDocument/UnitTestIndexTestDocumentFactoryF.cs @@ -0,0 +1,34 @@ +ο»Ώnamespace WebExpress.WebIndex.Test.Document +{ + /// + /// Factory class for creating unit test documents of type UnitTestIndexTestDocumentF. + /// + public class UnitTestIndexTestDocumentFactoryF : UnitTestIndexTestDocumentFactory + { + /// + /// Generates a list of test data for unit testing. + /// + /// + /// A list of objects. + /// + public static List GenerateTestData() + { + var testDataList = new List + { + new () + { + Id = Guid.Parse("c3d50744-5d66-422c-a0ab-ff024c1eacfc"), + Name = "😊🌸🐼" + }, + new () + { + Id = Guid.Parse("98714568-2925-46a0-b9ea-999a94ef07b4"), + Name = "张伟" + } + + }; + + return testDataList; + } + } +} diff --git a/src/WebExpress.WebIndex.Test/TestWqlExpressionNodeFilterFunctionConstant.cs b/src/WebExpress.WebIndex.Test/TestWqlExpressionNodeFilterFunctionConstant.cs new file mode 100644 index 0000000..192f986 --- /dev/null +++ b/src/WebExpress.WebIndex.Test/TestWqlExpressionNodeFilterFunctionConstant.cs @@ -0,0 +1,25 @@ +ο»Ώusing WebExpress.WebIndex.Wql.Function; + +namespace WebExpress.WebIndex.Test +{ + /// + /// Represents a test WQL expression node filter function constant for UnitTestIndexTestDocumentA. + /// + /// The type of the index item. + internal class TestWqlExpressionNodeFilterFunctionConstant : IWqlExpressionNodeFilterFunction where TIndexItem : IIndexItem + { + // + // Returns the name of the function. + // + public string Name => "Test"; + + /// + /// Executes the function. + /// + /// The return value. + public object Execute() + { + return "abc"; + } + } +} diff --git a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexAnalyze.cs b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexAnalyze.cs index cce07e8..64ee9d4 100644 --- a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexAnalyze.cs +++ b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexAnalyze.cs @@ -4,6 +4,9 @@ namespace WebExpress.WebIndex.Test.Token { + /// + /// A unit test class for analyzing tokens. + /// public class UnitTestIndexAnalyze : IClassFixture { /// @@ -17,7 +20,7 @@ public class UnitTestIndexAnalyze : IClassFixture protected UnitTestIndexFixtureToken Fixture { get; set; } /// - /// Constructor + /// Initializes a new instance of the class. /// /// The test context. /// The log. @@ -30,100 +33,44 @@ public UnitTestIndexAnalyze(UnitTestIndexFixtureToken fixture, ITestOutputHelper /// /// Tests the analysis function of an supported language. /// - [Fact] - public void Analyze1_En() + [Theory] + [InlineData("en", "abc def, ghi jkl mno-pip.", "abc", "def", "ghi", "jkl", "mno", "pip")] + [InlineData("en", "Be the change that you wish to see in the world. 😊🌸🐼", "change", "wish", "world")] + [InlineData("en", "??? ??")] + [InlineData("en", "???... ")] + [InlineData("en", "theya??r", "theya")] + [InlineData("en", "Life is like riding a bicycle. To keep your balance, you must keep moving.", "life", "riding", "bicycle", "balance", "moving")] + [InlineData("en", "β‰Ύβ‰ΏβŠ€βŠβŠ‚βŠƒβŠ„βŠ…βŠ†βŠ‡βŠˆβŠ‰")] + [InlineData("en", "β˜…*€’£Β₯Β©β–‘β–’β–“β”‚β”€β”œ")] + [InlineData("en", "Hello Helena, hello Helge!", "helena", "helge")] + [InlineData("en", "http://example.com/abc", "http", "example", "com", "abc")] + public void Token(string culture, string input, params string[] expected) { - // preconditions - var input = "abc def, ghi jkl mno-p."; - - // test execution - var tokens = Fixture.TokenAnalyzer.Analyze(input, CultureInfo.GetCultureInfo("en")); - - Assert.Equal(5, tokens.Count()); - Assert.True(tokens.First().Position == 0); - Assert.Equal("abc", tokens.First().Value); - Assert.True(tokens.Skip(1).First().Position == 1); - Assert.Equal("def", tokens.Skip(1).First().Value); - Assert.True(tokens.Skip(2).First().Position == 2); - Assert.Equal("ghi", tokens.Skip(2).First().Value); - Assert.True(tokens.Skip(3).First().Position == 3); - Assert.Equal("jkl", tokens.Skip(3).First().Value); - Assert.True(tokens.Skip(4).First().Position == 4); - Assert.Equal("mno-p", tokens.Skip(4).First().Value); - } - - /// - /// Tests the analysis function of an supported language. - /// - [Fact] - public void Analyze2_En() - { - // preconditions - var input = Fixture.GetRessource("JourneyThroughTheUniverse.en"); - // test execution - var tokens = Fixture.TokenAnalyzer.Analyze(input, CultureInfo.GetCultureInfo("en")); + var tokens = Fixture.TokenAnalyzer.Analyze(input, CultureInfo.GetCultureInfo(culture)); - Assert.Equal(241, tokens.Count()); // of 546 + Assert.Equal(expected, tokens.Select(x => x.Value)); } /// /// Tests the analysis function of an supported language. /// - [Fact] - public void Analyze3_En() - { - var input = Fixture.GetRessource("InterstellarConversations.en"); - - // test execution - var tokens = Fixture.TokenAnalyzer.Analyze(input, CultureInfo.GetCultureInfo("en")); - - Assert.Equal(170, tokens.Count()); // of 281 - } - - /// - /// Tests the analysis function of an supported language. - /// - [Fact] - public void Analyze_De() - { - // preconditions - var input = Fixture.GetRessource("BotanischeBindungenMicrosReiseZuVerdantia.de"); - - // test execution - var tokens = Fixture.TokenAnalyzer.Analyze(input, CultureInfo.GetCultureInfo("de")); - - Assert.Equal(418, tokens.Count()); // of 731 - } - - /// - /// Tests the analysis function of a regional language. - /// - [Fact] - public void Analyze_DeDE() - { - // preconditions - var input = Fixture.GetRessource("BotanischeBindungenMicrosReiseZuVerdantia.de"); - - // test execution - var tokens = Fixture.TokenAnalyzer.Analyze(input, CultureInfo.GetCultureInfo("de-DE")); - - Assert.Equal(418, tokens.Count()); // of 731 - } - - /// - /// Tests the analysis function of an unsupported language. - /// - [Fact] - public void Analyze_Fr() + [Theory] + [InlineData("en", "JourneyThroughTheUniverse.en", 241)] + [InlineData("en", "InterstellarConversations.en", 161)] + [InlineData("de", "BotanischeBindungenMicrosReiseZuVerdantia.de", 392)] + [InlineData("de-DE", "BotanischeBindungenMicrosReiseZuVerdantia.de", 392)] + [InlineData("fr", "BotanischeBindungenMicrosReiseZuVerdantia.de", 716)] + public void Ressource(string culture, string ressource, int count) { // preconditions - var input = Fixture.GetRessource("BotanischeBindungenMicrosReiseZuVerdantia.de"); + var input = Fixture.GetRessource(ressource); // test execution - var tokens = Fixture.TokenAnalyzer.Analyze(input, CultureInfo.GetCultureInfo("fr")); + var tokens = Fixture.TokenAnalyzer.Analyze(input, CultureInfo.GetCultureInfo(culture)); - Assert.Equal(719, tokens.Count()); // of 731 + Assert.Equal(count, tokens.Count()); + Assert.DoesNotContain(tokens.Select(x => x.Value), new object[] { "it", "or" }); } } } diff --git a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexAnalyzeNumber.cs b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexAnalyzeNumber.cs index 58724e5..5ad04eb 100644 --- a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexAnalyzeNumber.cs +++ b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexAnalyzeNumber.cs @@ -4,6 +4,10 @@ namespace WebExpress.WebIndex.Test.Token { + /// + /// Unit tests for analyzing numbers in different cultures. + /// + /// public class UnitTestIndexAnalyzeNumber : IClassFixture { /// @@ -17,7 +21,7 @@ public class UnitTestIndexAnalyzeNumber : IClassFixture - /// Constructor + /// Initializes a new instance of the class. /// /// The test context. /// The log. @@ -30,205 +34,179 @@ public UnitTestIndexAnalyzeNumber(UnitTestIndexFixtureToken fixture, ITestOutput /// /// Tests the number input. /// - [Fact] - public void Number_En() + [Theory] + [InlineData("1", 1, "en")] + [InlineData("1", 1, "de")] + [InlineData("-1", -1, "en")] + [InlineData("-1", -1, "de")] + public void Number(string term, double expected, string culture) { // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("1", CultureInfo.GetCultureInfo("en")); + var tokens = Fixture.TokenAnalyzer.Analyze(term, CultureInfo.GetCultureInfo(culture)); - Assert.Equal(1, (double)tokens.FirstOrDefault()?.Value); + Assert.Equal(expected, (double)tokens.FirstOrDefault()?.Value); } /// - /// Tests the number input. - /// - [Fact] - public void Number_De() - { - // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("1", CultureInfo.GetCultureInfo("de")); - - Assert.Equal(1, (double)tokens.FirstOrDefault()?.Value); - } - - /// - /// Tests the number input. - /// - [Fact] - public void NevativeNumber_En() - { - // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("-1", CultureInfo.GetCultureInfo("en")); - - Assert.Equal(-1, (double)tokens.FirstOrDefault()?.Value); - } - - /// - /// Tests the number input. - /// - [Fact] - public void NevativeNumber_De() - { - // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("-1", CultureInfo.GetCultureInfo("de")); - - Assert.Equal(-1, (double)tokens.FirstOrDefault()?.Value); - } - - /// - /// Tests the number input. - /// - [Fact] - public void InvalidNevativeNumber_En() - { - // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("-st", CultureInfo.GetCultureInfo("en")); - - Assert.Equal("st", tokens.FirstOrDefault()?.Value); - } - - /// - /// Tests the number input. - /// - [Fact] - public void InvalidNevativeNumber_De() - { - // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("-st", CultureInfo.GetCultureInfo("de")); - - Assert.Equal("st", tokens.FirstOrDefault()?.Value); - } - - /// - /// Tests the number input. + /// Tests the invalid number input. /// - [Fact] - public void Double_En() + [Theory] + [InlineData("-st", "st", "en")] + [InlineData("-st", "st", "de")] + [InlineData("10b0", "10 b0", "en")] + public void InvalidNumber(string term, string expected, string culture) { // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("1,0038.76", CultureInfo.GetCultureInfo("en")); + var tokens = Fixture.TokenAnalyzer.Analyze(term, CultureInfo.GetCultureInfo(culture)); - Assert.Equal(10038.76, (double)tokens.FirstOrDefault()?.Value); + Assert.Equal(expected, string.Join(" ", tokens.Select(x => x.Value))); } /// - /// Tests the number input. + /// Tests the double input. /// - [Fact] - public void Double_De() + [Theory] + [InlineData("10038.76", 10038.76, "en")] + [InlineData("1,0038.76", 10038.76, "en")] + [InlineData("10038,76", 10038.76, "de")] + [InlineData("1.0038,76", 10038.76, "de")] + [InlineData("-1,0038.76", -10038.76, "en")] + [InlineData("-1.0038,76", -10038.76, "de")] + public void Double(string term, double expected, string culture) { // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("1.0038,76", CultureInfo.GetCultureInfo("de")); + var tokens = Fixture.TokenAnalyzer.Analyze(term, CultureInfo.GetCultureInfo(culture)); - Assert.Equal(10038.76, (double)tokens.FirstOrDefault()?.Value); + Assert.Equal(expected, (double)tokens.FirstOrDefault()?.Value); } /// /// Tests the number input. /// - [Fact] - public void InvalidDouble_En() + [Theory] + [InlineData("1.0038,76", new double[] { 1.0038, 76 }, "en")] + [InlineData("1,0038.76", new double[] { 1.0038, 76 }, "de")] + public void InvalidDouble(string term, double[] expected, string culture) { // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("1.0038,76", CultureInfo.GetCultureInfo("en")); + var tokens = Fixture.TokenAnalyzer.Analyze(term, CultureInfo.GetCultureInfo(culture)); - Assert.Equal("1.0038,76", tokens.FirstOrDefault()?.Value); - } - - /// - /// Tests the number input. - /// - [Fact] - public void InvalidDouble_De() - { - // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("1,0038.76", CultureInfo.GetCultureInfo("de")); - - Assert.Equal(10038.76, (double)tokens.FirstOrDefault()?.Value); + Assert.Equal(expected, tokens.Select(x => (double)x.Value)); } /// /// Tests the number with exponent input. /// - [Fact] - public void Exponent_En() + [Theory] + [InlineData("1.0038e76", 1.0038e76, "en")] + [InlineData("1.0038E76", 1.0038e76, "en")] + [InlineData("1.0038e+76", 1.0038e76, "en")] + [InlineData("1.0038E+76", 1.0038e76, "en")] + [InlineData("1.0038e-76", 1.0038e-76, "en")] + [InlineData("1.0038E-76", 1.0038e-76, "en")] + [InlineData("1,0038e76", 1.0038e76, "de")] + [InlineData("+1.0038e76", 1.0038e76, "en")] + [InlineData("+1,0038e76", 1.0038e76, "de")] + [InlineData("-1.0038e76", -1.0038e76, "en")] + [InlineData("-1,0038e76", -1.0038e76, "de")] + public void Exponent(string term, double expected, string culture) { // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("1.0038e76", CultureInfo.GetCultureInfo("en")); + var tokens = Fixture.TokenAnalyzer.Analyze(term, CultureInfo.GetCultureInfo(culture)); - Assert.Equal(1.0038E+76, (double)tokens.FirstOrDefault()?.Value); + Assert.Equal(expected, (double)tokens.FirstOrDefault()?.Value); } /// /// Tests the number with exponent input. /// - [Fact] - public void Exponent_De() + [Theory] + [InlineData("∞", double.PositiveInfinity, "en")] + [InlineData("-∞", double.NegativeInfinity, "en")] + [InlineData("∞", double.PositiveInfinity, "de")] + [InlineData("-∞", double.NegativeInfinity, "de")] + public void Infinity(string term, double expected, string culture) { // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("1,0038e76", CultureInfo.GetCultureInfo("de")); + var tokens = Fixture.TokenAnalyzer.Analyze(term, CultureInfo.GetCultureInfo(culture)); - Assert.Equal(1.0038E+76, (double)tokens.FirstOrDefault()?.Value); + Assert.Equal(expected, (double)tokens.FirstOrDefault()?.Value); } /// - /// Tests the negative number input. + /// Tests the add with input. /// - [Fact] - public void NevativeExponent_En() + [Theory] + [InlineData("2+3", new double[] { 2, 3 }, "en")] + [InlineData("2 + 3", new double[] { 2, 3 }, "en")] + [InlineData("2+3", new double[] { 2, 3 }, "de")] + [InlineData("2 + 3", new double[] { 2, 3 }, "de")] + public void Add(string term, double[] expected, string culture) { // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("1e-76", CultureInfo.GetCultureInfo("en")); + var tokens = Fixture.TokenAnalyzer.Analyze(term, CultureInfo.GetCultureInfo(culture)); - Assert.Equal(1E-76, (double)tokens.FirstOrDefault()?.Value); + Assert.Equal(expected, tokens.Select(x => (double)x.Value)); } /// - /// Tests the number input. + /// Tests the minus with input. /// - [Fact] - public void NevativeExponent_De() + [Theory] + [InlineData("2-3", new double[] { 2, 3 }, "en")] + [InlineData("2 - 3", new double[] { 2, 3 }, "en")] + [InlineData("2-3", new double[] { 2, 3 }, "de")] + [InlineData("2 - 3", new double[] { 2, 3 }, "de")] + public void Minus(string term, double[] expected, string culture) { // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("1e-76", CultureInfo.GetCultureInfo("de")); + var tokens = Fixture.TokenAnalyzer.Analyze(term, CultureInfo.GetCultureInfo(culture)); - Assert.Equal(1E-76, (double)tokens.FirstOrDefault()?.Value); + Assert.Equal(expected, tokens.Select(x => (double)x.Value)); } /// - /// Tests the text_number input. + /// Tests the power with input. /// - [Fact] - public void TextWithNumber_En1() + [Theory] + [InlineData("2^3", new double[] { 2, 3 }, "en")] + [InlineData("2 ^ 3", new double[] { 2, 3 }, "en")] + [InlineData("2^3", new double[] { 2, 3 }, "de")] + [InlineData("2 ^ 3", new double[] { 2, 3 }, "de")] + public void Power(string term, double[] expected, string culture) { // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("N_1", CultureInfo.GetCultureInfo("en")); + var tokens = Fixture.TokenAnalyzer.Analyze(term, CultureInfo.GetCultureInfo(culture)); - Assert.Equal("n_1", tokens.FirstOrDefault()?.Value); + Assert.Equal(expected, tokens.Select(x => (double)x.Value)); } /// /// Tests the text_number input. /// - [Fact] - public void TextWithNumber_En2() + [Theory] + [InlineData("N1", "n1", "en")] + [InlineData("N1", "n1", "de")] + public void TextWithNumber(string term, string expected, string culture) { // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("N1", CultureInfo.GetCultureInfo("en")); + var tokens = Fixture.TokenAnalyzer.Analyze(term, CultureInfo.GetCultureInfo(culture)); - Assert.Equal("n1", tokens.FirstOrDefault()?.Value); + Assert.Equal(expected, tokens.FirstOrDefault()?.Value); } /// - /// Tests the wildcard input. + /// Tests the text_number with wildcatd input. /// - [Fact] - public void NumberWithWildcard_En() + [Theory] + [InlineData("Name?23", "name?23", "en")] + [InlineData("Name?23", "name?23", "de")] + public void NumberWithWildcard(string term, string expected, string culture) { // test execution - var tokens = Fixture.TokenAnalyzer.Analyze("Name_?23", CultureInfo.GetCultureInfo("en"), true); + var tokens = Fixture.TokenAnalyzer.Analyze(term, CultureInfo.GetCultureInfo(culture), true); - Assert.Equal("name_?23", tokens.FirstOrDefault()?.Value); + Assert.Equal(expected, tokens.FirstOrDefault()?.Value); } } } diff --git a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageEmpty.cs b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageEmpty.cs index df8cc54..a488845 100644 --- a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageEmpty.cs +++ b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageEmpty.cs @@ -19,7 +19,7 @@ public class UnitTestIndexPipeStageEmpty : IClassFixture - /// Constructor + /// Initializes a new instance of the class. /// /// The test context. /// The log. diff --git a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageLowerCase.cs b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageLowerCase.cs index a8e1030..d21825e 100644 --- a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageLowerCase.cs +++ b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageLowerCase.cs @@ -6,6 +6,12 @@ namespace WebExpress.WebIndex.Test.Token { + /// + /// Unit tests for the IndexPipeStageLowerCase class. + /// + /// + /// This class tests the functionality of converting terms to lower case as part of the stemming process. + /// public class UnitTestIndexPipeStageLowerCase : IClassFixture { /// @@ -19,7 +25,7 @@ public class UnitTestIndexPipeStageLowerCase : IClassFixture - /// Constructor + /// Initializes a new instance of the class. /// /// The test context. /// The log. @@ -32,25 +38,22 @@ public UnitTestIndexPipeStageLowerCase(UnitTestIndexFixtureToken fixture, ITestO /// /// Tests the lower case method. This function is part of the stemming process and convert characters in lower case. /// - [Fact] - public void LowerCase() + [Theory] + [InlineData("Babies", "babies", "en")] + [InlineData("Cities", "cities", "en")] + [InlineData("Countries", "countries", "en")] + [InlineData("Families", "families", "en")] + public void LowerCase(string term, string normalizeTerm, string cultureString) { - var culture = CultureInfo.GetCultureInfo("en"); + // preconditions + var culture = CultureInfo.GetCultureInfo(cultureString); var pipeStage = new IndexPipeStageConverterLowerCase(Fixture.Context); - (string, string)[] words = - [ - ("Babies", "babies"), - ("Cities", "cities"), - ("Countries", "countries"), - ("Families", "families") - ]; + // test execution + var res = pipeStage.Process(IndexTermTokenizer.Tokenize(term, culture), culture) + .FirstOrDefault(); - var res = pipeStage.Process(IndexTermTokenizer.Tokenize(string.Join(" ", words.Select(x => x.Item1)), culture), culture) - .Select(x => x.Value) - .ToList(); - - Assert.True(res.Intersect(words.Select(x => x.Item2)).Count() == res.Count); + Assert.Equal(normalizeTerm, res.Value); } } } diff --git a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageMisspelled.cs b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageMisspelled.cs index 747e634..76a20ef 100644 --- a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageMisspelled.cs +++ b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageMisspelled.cs @@ -19,7 +19,7 @@ public class UnitTestIndexPipeStageMisspelled : IClassFixture - /// Constructor + /// Initializes a new instance of the class. /// /// The test context. /// The log. diff --git a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageNormalizer.cs b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageNormalizer.cs index 5139428..61e3029 100644 --- a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageNormalizer.cs +++ b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageNormalizer.cs @@ -19,7 +19,7 @@ public class UnitTestIndexPipeStageNormalizer : IClassFixture - /// Constructor + /// Initializes a new instance of the class. /// /// The test context. /// The log. @@ -32,31 +32,29 @@ public UnitTestIndexPipeStageNormalizer(UnitTestIndexFixtureToken fixture, ITest /// /// Tests the Normalize method. This function is part of the stemming process and normalize terms. /// - [Fact] - public void Normalize() + [Theory] + [InlineData("rΓ©sumΓ©", "resume", "en")] + [InlineData("MΓ«tΓ l", "Metal", "en")] + [InlineData("Γ©lΓ©gant", "elegant", "en")] + [InlineData("clichΓ©", "cliche", "en")] + [InlineData("naΓ―ve", "naive", "en")] + [InlineData("soufflΓ©", "souffle", "en")] + [InlineData("dΓ©jΓ -vu", "deja vu", "en")] + [InlineData("tΓͺte-Γ -tΓͺte", "tete a tete", "en")] + [InlineData("SΓ£o-Paulo", "Sao Paulo", "en")] + [InlineData("BjΓΆrk", "Bjork", "en")] + public void Normalize(string term, string expected, string cultureString) { - var culture = CultureInfo.GetCultureInfo("en"); + // preconditions + var culture = CultureInfo.GetCultureInfo(cultureString); var pipeStage = new IndexPipeStageConverterNormalizer(Fixture.Context); - (string, string)[] words = - [ - ("rΓ©sumΓ©", "resume"), - ("MΓ«tΓ l", "Metal"), - ("Γ©lΓ©gant", "elegant"), - ("clichΓ©", "cliche"), - ("naΓ―ve", "naive"), - ("soufflΓ©", "souffle"), - ("dΓ©jΓ -vu", "deja-vu"), - ("tΓͺte-Γ -tΓͺte", "tete-a-tete"), - ("SΓ£o-Paulo", "Sao-Paulo"), - ("BjΓΆrk", "Bjork") - ]; - - var res = pipeStage.Process(IndexTermTokenizer.Tokenize(string.Join(" ", words.Select(x => x.Item1)), culture), culture) + // test execution + var res = pipeStage.Process(IndexTermTokenizer.Tokenize(term, culture), culture) .Select(x => x.Value) .ToList(); - Assert.True(res.Intersect(words.Select(x => x.Item2)).Count() == res.Count); + Assert.Equal(expected, string.Join(" ", res)); } } } diff --git a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageSingular.cs b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageSingular.cs index d988514..ccf9fda 100644 --- a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageSingular.cs +++ b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageSingular.cs @@ -19,7 +19,7 @@ public class UnitTestIndexPipeStageSingular : IClassFixture - /// Constructor + /// Initializes a new instance of the class. /// /// The test context. /// The log. @@ -32,73 +32,39 @@ public UnitTestIndexPipeStageSingular(UnitTestIndexFixtureToken fixture, ITestOu /// /// Tests the singular method. This function is part of the lemmatization process and the transformation of the token into the singular. /// - [Fact] - public void PluralToSingular_En() + [Theory] + // regular nouns (ies, ses xes, s) + [InlineData("babies", "baby", "en")] + [InlineData("countries", "country", "en")] + [InlineData("families", "family", "en")] + [InlineData("parties", "party", "en")] + [InlineData("pennies", "penny", "en")] + [InlineData("studies", "study", "en")] + [InlineData("stories", "story", "en")] + [InlineData("autos", "auto", "de")] + [InlineData("frauen", "frau", "de")] + [InlineData("kinder", "kind", "de")] + [InlineData("tische", "tisch", "de")] + // irregular nouns + [InlineData("axes", "axis", "en")] + [InlineData("indices", "index", "en")] + [InlineData("selves", "self", "en")] + [InlineData("vortexes", "vortex", "en")] + [InlineData("atlanten", "atlas", "de")] + [InlineData("bΓΌcher", "buch", "de")] + [InlineData("mΓ€nner", "mann", "de")] + [InlineData("stΓΌhle", "stuhl", "de")] + public void PluralToSingular(string pluralWord, string singularWord, string cultureString) { - var culture = CultureInfo.GetCultureInfo("en"); + // preconditions + var culture = CultureInfo.GetCultureInfo(cultureString); var pipeStage = new IndexPipeStageConverterSingular(Fixture.Context); - (string, string)[] words = - [ - // regular nouns (ies, ses xes, s) - ("babies", "baby"), - ("cities", "city"), - ("countries", "country"), - ("families", "family"), - ("parties", "party"), - ("pennies", "penny"), - ("studies", "study"), - ("stories", "story"), - // irregular nouns - ("axes", "axis"), - ("indices", "index"), - ("selves", "self"), - ("vortexes", "vortex") - ]; + // test execution + var res = pipeStage.Process(IndexTermTokenizer.Tokenize(pluralWord, culture), culture) + .FirstOrDefault(); - - var pluralWords = words.Select(x => x.Item1); - var singularWords = words.Select(x => x.Item2); - - var res = pipeStage.Process(IndexTermTokenizer.Tokenize(string.Join(" ", pluralWords), culture), culture) - .Select(x => x.Value) - .ToList(); - - Assert.True(res.Intersect(singularWords).Count() == res.Count); - } - - /// - /// Tests the singular method. This function is part of the lemmatization process and the transformation of the token into the singular. - /// - [Fact] - public void PluralToSingular_De() - { - var culture = CultureInfo.GetCultureInfo("de"); - var pipeStage = new IndexPipeStageConverterSingular(Fixture.Context); - - (string, string)[] words = - [ - // regular nouns (en, er, e, s) - ("autos", "auto"), - ("frauen", "frau"), - ("kinder", "kind"), - ("tische", "tisch"), - // irregular nouns - ("atlanten", "atlas"), - ("bΓΌcher", "buch"), - ("mΓ€nner", "mann"), - ("stΓΌhle", "stuhl") - ]; - - - var pluralWords = words.Select(x => x.Item1); - var singularWords = words.Select(x => x.Item2); - - var res = pipeStage.Process(IndexTermTokenizer.Tokenize(string.Join(" ", pluralWords), culture), culture) - .Select(x => x.Value) - .ToList(); - - Assert.True(res.Intersect(singularWords).Count() == res.Count); + Assert.Equal(singularWord, res?.Value); } } } diff --git a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageStopWord.cs b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageStopWord.cs index 1cd9162..2a36c31 100644 --- a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageStopWord.cs +++ b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageStopWord.cs @@ -19,7 +19,7 @@ public class UnitTestIndexPipeStageStopWord : IClassFixture - /// Constructor + /// Initializes a new instance of the class. /// /// The test context. /// The log. @@ -32,51 +32,51 @@ public UnitTestIndexPipeStageStopWord(UnitTestIndexFixtureToken fixture, ITestOu /// /// Tests the stop word method. This function is part of the stemming process and removes stop words. /// - [Fact] - public void StopWord_En() + [InlineData + ( + "en", + "May the force be with you.", + "may", "the", "be", "with", "you" + )] + [InlineData + ( + "en", + "Would you like tea or coffee?", + "would", "you", "like", "or", "coffee" + )] + [InlineData + ( + "en", + "If it rains tomorrow, we will stay indoors.", + "if", "it", "we", "will", "stay" + )] + [Theory] + [InlineData + ( + "de", + "Als Gregor Samsa eines Morgens aus unruhigen TrΓ€umen erwachte, fand er sich in seinem Bett zu einem ungeheueren Ungeziefer verwandelt.", + "als", "eines", "aus", "er", "sich", "in", "seinem", "zu", "einem" + )] + [InlineData + ( + "en", + "😊🌸🐼", + null + )] + public void StopWord(string cultureStr, string str, params string[] tokenStr) { - var culture = CultureInfo.GetCultureInfo("en"); + // preconditions + var culture = CultureInfo.GetCultureInfo(cultureStr); var pipeStage = new IndexPipeStageFilterStopWord(Fixture.Context); - var token = IndexTermTokenizer.Tokenize("May the force be with you.", culture); + var token = IndexTermTokenizer.Tokenize(str.ToLower(), culture); + // test execution var res = pipeStage.Process(token, culture) .Select(x => x.Value) .ToList(); - Assert.DoesNotContain("the", res); - Assert.DoesNotContain("be", res); - Assert.DoesNotContain("with", res); - - Assert.True(token.Count() - 3 == res.Count); - } - - /// - /// Tests the stop word method. This function is part of the stemming process and removes stop words. - /// - [Fact] - public void StopWord_De() - { - var culture = CultureInfo.GetCultureInfo("de"); - var pipeStage = new IndexPipeStageFilterStopWord(Fixture.Context); - - var token = IndexTermTokenizer.Tokenize("Als Gregor Samsa eines Morgens aus unruhigen TrΓ€umen erwachte, fand er sich in seinem Bett zu einem ungeheueren Ungeziefer verwandelt.".ToLower(), culture); - - var res = pipeStage.Process(token, culture) - .Select(x => x.Value) - .ToList(); - - Assert.DoesNotContain("als", res); - Assert.DoesNotContain("eines", res); - Assert.DoesNotContain("aus", res); - Assert.DoesNotContain("er", res); - Assert.DoesNotContain("sich", res); - Assert.DoesNotContain("in", res); - Assert.DoesNotContain("seinem", res); - Assert.DoesNotContain("zu", res); - Assert.DoesNotContain("einem", res); - - Assert.True(token.Count() - 9 == res.Count); + Assert.DoesNotContain(tokenStr, res); } } } diff --git a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageSurrogateCharacter.cs b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageSurrogateCharacter.cs new file mode 100644 index 0000000..3a7835e --- /dev/null +++ b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageSurrogateCharacter.cs @@ -0,0 +1,53 @@ +ο»Ώusing System.Globalization; +using WebExpress.WebIndex.Term; +using WebExpress.WebIndex.Term.Pipeline; +using WebExpress.WebIndex.Test.Fixture; +using Xunit.Abstractions; + +namespace WebExpress.WebIndex.Test.Token +{ + public class UnitTestIndexPipeStageSurrogateCharacter : IClassFixture + { + /// + /// Returns the log. + /// + public ITestOutputHelper Output { get; private set; } + + /// + /// Returns the test context. + /// + protected UnitTestIndexFixtureToken Fixture { get; set; } + + /// + /// Initializes a new instance of the class. + /// + /// The test context. + /// The log. + public UnitTestIndexPipeStageSurrogateCharacter(UnitTestIndexFixtureToken fixture, ITestOutputHelper output) + { + Fixture = fixture; + Output = output; + } + + /// + /// Tests the surrogate character method. This function is part of the stemming process and removes words with surrogate characters. + /// + [Fact] + public void Surrogate() + { + var culture = CultureInfo.GetCultureInfo("en"); + var pipeStage = new IndexPipeStageFilterSurrogateCharacter(Fixture.Context); + + var chars = new char[] { '\uD800', '\uDC00' }; // this is a surrogate pair + var token = IndexTermTokenizer.Tokenize($"a surrogate pair like this '{new string(chars)}' must be removed.", culture); + + var res = pipeStage.Process(token, culture) + .Select(x => x.Value) + .ToList(); + + Assert.DoesNotContain(new string(chars), res); + + Assert.True(token.Count() - 1 == res.Count); + } + } +} diff --git a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageSynonym.cs b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageSynonym.cs index dffcd30..e8682ec 100644 --- a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageSynonym.cs +++ b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageSynonym.cs @@ -19,7 +19,7 @@ public class UnitTestIndexPipeStageSynonym : IClassFixture - /// Constructor + /// Initializes a new instance of the class. /// /// The test context. /// The log. @@ -32,43 +32,20 @@ public UnitTestIndexPipeStageSynonym(UnitTestIndexFixtureToken fixture, ITestOut /// /// Tests the synonym method. This function is part of the lemmatization process and reduced sysnonyms. /// - [Fact] - public void Synonym_En() + [Theory] + [InlineData("joyful", "happy", "en")] + [InlineData("kfz", "auto", "de")] + public void Synonym(string synonymWord, string normalWord, string cultureString) { - var culture = CultureInfo.GetCultureInfo("en"); + // preconditions + var culture = CultureInfo.GetCultureInfo(cultureString); var pipeStage = new IndexPipeStageConverterSynonym(Fixture.Context); - (string, string)[] words = - [ - ("joyful", "happy") - ]; + // test execution + var res = pipeStage.Process(IndexTermTokenizer.Tokenize(synonymWord, culture), culture) + .FirstOrDefault(); - var res = pipeStage.Process(IndexTermTokenizer.Tokenize(string.Join(" ", words.Select(x => x.Item1)), culture), culture) - .Select(x => x.Value) - .ToList(); - - Assert.Equal(words.Select(x => x.Item2), res); - } - - /// - /// Tests the synonym method. This function is part of the lemmatization process and reduced sysnonyms. - /// - [Fact] - public void Synonym_De() - { - var culture = CultureInfo.GetCultureInfo("de"); - var pipeStage = new IndexPipeStageConverterSynonym(Fixture.Context); - - (string, string)[] words = - [ - ("kfz", "auto") - ]; - - var res = pipeStage.Process(IndexTermTokenizer.Tokenize(string.Join(" ", words.Select(x => x.Item1)), culture), culture) - .Select(x => x.Value) - .ToList(); - - Assert.Equal(words.Select(x => x.Item2), res); + Assert.Equal(normalWord, res?.Value); } } -} +} \ No newline at end of file diff --git a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageTrim.cs b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageTrim.cs index 779141c..8478d45 100644 --- a/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageTrim.cs +++ b/src/WebExpress.WebIndex.Test/Token/UnitTestIndexPipeStageTrim.cs @@ -19,7 +19,7 @@ public class UnitTestIndexPipeStageTrim : IClassFixture - /// Constructor + /// Initializes a new instance of the class. /// /// The test context. /// The log. diff --git a/src/WebExpress.WebIndex.Test/WQL/UnitTestWqlSearchFuzzyA.cs b/src/WebExpress.WebIndex.Test/WQL/UnitTestWqlSearchFuzzyA.cs index c4a4183..567e01e 100644 --- a/src/WebExpress.WebIndex.Test/WQL/UnitTestWqlSearchFuzzyA.cs +++ b/src/WebExpress.WebIndex.Test/WQL/UnitTestWqlSearchFuzzyA.cs @@ -120,8 +120,8 @@ public void Fuzzy() Assert.NotNull(res); Assert.NotNull(item); - Assert.Equal(3, res.Count()); - Assert.Equal("Text = 'Hel' ~50", wql.ToString()); + Assert.Equal(4, res.Count()); + Assert.Equal("Text ~ 'Helena' ~50", wql.ToString()); Assert.NotNull(wql.Filter); Assert.Null(wql.Order); Assert.Null(wql.Partitioning); @@ -140,8 +140,8 @@ public void FuzzyFromQueryable() Assert.NotNull(res); Assert.NotNull(item); - Assert.Equal(3, res.Count()); - Assert.Equal("Text = 'Hel~' ~50", wql.ToString()); + Assert.Equal(6, res.Count()); + Assert.Equal("Text ~ 'Hel' ~50", wql.ToString()); Assert.NotNull(wql.Filter); Assert.Null(wql.Order); Assert.Null(wql.Partitioning); diff --git a/src/WebExpress.WebIndex.Test/WQL/UnitTestWqlSearchPhraseA.cs b/src/WebExpress.WebIndex.Test/WQL/UnitTestWqlSearchPhraseA.cs index 14d3858..c86fd35 100644 --- a/src/WebExpress.WebIndex.Test/WQL/UnitTestWqlSearchPhraseA.cs +++ b/src/WebExpress.WebIndex.Test/WQL/UnitTestWqlSearchPhraseA.cs @@ -1,5 +1,4 @@ -ο»Ώusing WebExpress.WebIndex.Test.Document; -using WebExpress.WebIndex.Test.Fixture; +ο»Ώusing WebExpress.WebIndex.Test.Fixture; using Xunit.Abstractions; namespace WebExpress.WebIndex.Test.WQL @@ -154,7 +153,7 @@ public void MultipleMatch2() Assert.Equal(1, res.Count()); Assert.Contains("c7d8f9e0-3a2b-4c5d-8e6f-9a1b0c2d4e5f", res.Select(x => x.Id.ToString())); } - + /// /// Tests phrase search, which retrieves content from documents that contain a specific order and combination of words defined by the phrase. /// The test produces an unexpected positive result even though there is no exact match. This is because stop words are removed but are @@ -164,7 +163,8 @@ public void MultipleMatch2() public void UnexpectedMatch() { // test execution - var wql = Fixture.ExecuteWql("text='Hello Helena, hello Helge!'"); + // Hello Helena and Helge! + var wql = Fixture.ExecuteWql("text='Hello Helena hello Helge!'"); var res = wql?.Apply(); Assert.NotNull(res); diff --git a/src/WebExpress.WebIndex.Test/WQL/UnitTestWqlSearchPhraseB.cs b/src/WebExpress.WebIndex.Test/WQL/UnitTestWqlSearchPhraseB.cs index a02a6ae..1e975b1 100644 --- a/src/WebExpress.WebIndex.Test/WQL/UnitTestWqlSearchPhraseB.cs +++ b/src/WebExpress.WebIndex.Test/WQL/UnitTestWqlSearchPhraseB.cs @@ -1,18 +1,12 @@ ο»Ώusing WebExpress.WebIndex.Test.Fixture; -using Xunit.Abstractions; namespace WebExpress.WebIndex.Test.WQL { /// /// Phrase search (exact word sequence) /// - public class UnitTestWqlSearchPhraseB(UnitTestIndexFixtureWqlB fixture, ITestOutputHelper output) : IClassFixture + public class UnitTestWqlSearchPhraseB(UnitTestIndexFixtureWqlB fixture) : IClassFixture { - /// - /// Returns the log. - /// - public ITestOutputHelper Output { get; private set; } = output; - /// /// Returns the test context. /// @@ -21,94 +15,52 @@ public class UnitTestWqlSearchPhraseB(UnitTestIndexFixtureWqlB fixture, ITestOut /// /// Tests the parser. /// - [Fact] - public void ParseValidWql1() + [Theory] + [InlineData("Description='lorem ipsum'")] + [InlineData("description='lorem ipsum'")] + [InlineData("description=\"lorem ipsum\"")] + [InlineData("description=lorem")] + [InlineData("Adress.Street = lorem")] + [InlineData("adress.street=lorem")] + public void ParseValidWql(string wqlString) { // test execution - var wql = Fixture.ExecuteWql("description='lorem ipsum'"); - Assert.False(wql.HasErrors); - } + var wql = Fixture.ExecuteWql(wqlString); - /// - /// Tests the parser. - /// - [Fact] - public void ParseValidWql2() - { - // test execution - var wql = Fixture.ExecuteWql("description=\"lorem ipsum\""); Assert.False(wql.HasErrors); } /// /// Tests the parser. /// - [Fact] - public void ParseValidWql3() - { - // test execution - var wql = Fixture.ExecuteWql("description=lorem"); - Assert.False(wql.HasErrors); - } - - /// - /// Tests the parser. - /// - [Fact] - public void ParseInvalidWql1() - { - // test execution - var wql = Fixture.ExecuteWql("description=lorem ipsum"); - Assert.True(wql.HasErrors); - } - - /// - /// Tests the parser. - /// - [Fact] - public void ParseInvalidWql2() + [Theory] + [InlineData("description=lorem ipsum")] + [InlineData("description='lorem ipsum")] + [InlineData("description='lorem ipsum\"")] + [InlineData("='lorem ipsum'")] + [InlineData("Adress,Street=lorem")] + public void ParseInvalidWql(string wqlString) { // test execution - var wql = Fixture.ExecuteWql("description='lorem ipsum"); - Assert.True(wql.HasErrors); - } - - /// - /// Tests the parser. - /// - [Fact] - public void ParseInvalidWql3() - { - // test execution - var wql = Fixture.ExecuteWql("description='lorem ipsum\""); - Assert.True(wql.HasErrors); - } - - /// - /// Tests the parser. - /// - [Fact] - public void ParseInvalidWql4() - { - // test execution - var wql = Fixture.ExecuteWql("='lorem ipsum'"); + var wql = Fixture.ExecuteWql(wqlString); Assert.True(wql.HasErrors); } /// /// Tests phrase search, which retrieves content from documents that contain a specific order and combination of words defined by the phrase. /// - [Fact] - public void SingleMatch1() + [Theory] + [InlineData("Description='lorem'", "lorem")] + public void SingleMatch(string wqlString, string expected) { // test execution - var wql = Fixture.ExecuteWql("Description='lorem'"); + var wql = Fixture.ExecuteWql(wqlString); var res = wql?.Apply(); Assert.NotNull(res); foreach (var description in res.Select(x => x.Description)) { - Assert.Contains("lorem", description); + Assert.Contains(expected, description); } } @@ -116,7 +68,7 @@ public void SingleMatch1() /// Tests phrase search, which retrieves content from documents that contain a specific order and combination of words defined by the phrase. /// [Fact] - public void MultipleMatch1() + public void MultipleMatch() { // test execution var wql = Fixture.ExecuteWql("Description='lorem ipsum'"); diff --git a/src/WebExpress.WebIndex.Test/WebExpress.WebIndex.Test.csproj b/src/WebExpress.WebIndex.Test/WebExpress.WebIndex.Test.csproj index 1ad76b3..4e8c28e 100644 --- a/src/WebExpress.WebIndex.Test/WebExpress.WebIndex.Test.csproj +++ b/src/WebExpress.WebIndex.Test/WebExpress.WebIndex.Test.csproj @@ -1,7 +1,7 @@ - net8.0 + net9.0 enable false true @@ -23,9 +23,13 @@ - - - + + + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/WebExpress.WebIndex.Wi/ArgumentParser.cs b/src/WebExpress.WebIndex.Wi/ArgumentParser.cs index eb112f7..373106b 100644 --- a/src/WebExpress.WebIndex.Wi/ArgumentParser.cs +++ b/src/WebExpress.WebIndex.Wi/ArgumentParser.cs @@ -32,7 +32,7 @@ public static ArgumentParser Current } /// - /// Constructor + /// Initializes a new instance of the class. /// public ArgumentParser() { diff --git a/src/WebExpress.WebIndex.Wi/IndexManager.cs b/src/WebExpress.WebIndex.Wi/IndexManager.cs index 70505ad..be20028 100644 --- a/src/WebExpress.WebIndex.Wi/IndexManager.cs +++ b/src/WebExpress.WebIndex.Wi/IndexManager.cs @@ -9,7 +9,7 @@ namespace WebExpress.WebIndex.Wi internal class IndexManager : WebIndex.IndexManager { /// - /// Constructor + /// Initializes a new instance of the class. /// public IndexManager() { diff --git a/src/WebExpress.WebIndex.Wi/Model/ObjectType.cs b/src/WebExpress.WebIndex.Wi/Model/ObjectType.cs index 53150d1..2f24c6f 100644 --- a/src/WebExpress.WebIndex.Wi/Model/ObjectType.cs +++ b/src/WebExpress.WebIndex.Wi/Model/ObjectType.cs @@ -9,7 +9,7 @@ namespace WebExpress.WebIndex.Wi.Model /// internal class ObjectType { - private Dictionary _typeCache = new Dictionary(); + private Dictionary _typeCache = []; /// /// Returns or sets the name of the object. diff --git a/src/WebExpress.WebIndex.Wi/Model/ViewModel.cs b/src/WebExpress.WebIndex.Wi/Model/ViewModel.cs index 457daf8..849f223 100644 --- a/src/WebExpress.WebIndex.Wi/Model/ViewModel.cs +++ b/src/WebExpress.WebIndex.Wi/Model/ViewModel.cs @@ -6,6 +6,9 @@ namespace WebExpress.WebIndex.Wi.Model { + /// + /// Represents the ViewModel for managing the indexing of project objects. + /// internal class ViewModel { /// @@ -56,7 +59,10 @@ public bool CreateIndexFile(string indexFile) var runtimeClass = CurrentObjectType.BuildRuntimeClass(); var context = new IndexContext { IndexDirectory = CurrentDirectory }; IndexManager = new IndexManager(); - IndexManager.Initialization(context); + + // use reflection to call the protected Initialization method + var method = typeof(IndexManager).GetMethod("Initialization", BindingFlags.Instance | BindingFlags.NonPublic); + method.Invoke(IndexManager, [context]); IndexManager.Create(runtimeClass, CultureInfo.GetCultureInfo("en"), IndexType.Storage); @@ -81,7 +87,10 @@ public bool OpenIndexFile(string indexFile) var context = new IndexContext { IndexDirectory = CurrentDirectory }; IndexManager = new IndexManager(); - IndexManager.Initialization(context); + + // use reflection to call the protected Initialization method + var method = typeof(IndexManager).GetMethod("Initialization", BindingFlags.Instance | BindingFlags.NonPublic); + method.Invoke(IndexManager, [context]); IndexManager.Create(runtimeClass, CultureInfo.GetCultureInfo("en"), IndexType.Storage); @@ -137,10 +146,11 @@ public bool DropIndexFile() public IEnumerable<(string, uint, uint, uint, IEnumerable)> GetIndexTerms() { var runtimeClass = CurrentObjectType.BuildRuntimeClass(); - var document = WiApp.ViewModel.IndexManager.GetIndexDocument(runtimeClass); + var document = IndexManager.GetIndexDocument(runtimeClass); var fieldProperty = runtimeClass.GetProperty(CurrentIndexField?.Name); + var fieldData = new IndexFieldData(fieldProperty); var methodInfo = document.GetType().GetMethod("GetReverseIndex"); - var reverseIndex = methodInfo.Invoke(document, [fieldProperty]); + var reverseIndex = methodInfo.Invoke(document, [fieldData]); var termProperty = reverseIndex.GetType().GetProperty("Term"); var term = termProperty.GetValue(reverseIndex) as IndexStorageSegmentTerm; diff --git a/src/WebExpress.WebIndex.Wi/WebExpress.WebIndex.Wi.csproj b/src/WebExpress.WebIndex.Wi/WebExpress.WebIndex.Wi.csproj index 702a8d9..5f9ef57 100644 --- a/src/WebExpress.WebIndex.Wi/WebExpress.WebIndex.Wi.csproj +++ b/src/WebExpress.WebIndex.Wi/WebExpress.WebIndex.Wi.csproj @@ -2,7 +2,7 @@ Exe - net8.0 + net9.0 enable disable WiApp diff --git a/src/WebExpress.WebIndex.WiUI/App.xaml b/src/WebExpress.WebIndex.WiUI/App.xaml new file mode 100644 index 0000000..47b32a1 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/App.xaml @@ -0,0 +1,19 @@ +ο»Ώ + + + + + + + + + + + + + diff --git a/src/WebExpress.WebIndex.WiUI/App.xaml.cs b/src/WebExpress.WebIndex.WiUI/App.xaml.cs new file mode 100644 index 0000000..032e9aa --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/App.xaml.cs @@ -0,0 +1,15 @@ +ο»Ώnamespace WebExpress.WebIndex.WiUI +{ + public partial class App : Application + { + public App() + { + InitializeComponent(); + } + + protected override Window CreateWindow(IActivationState? activationState) + { + return new Window(new AppShell()); + } + } +} \ No newline at end of file diff --git a/src/WebExpress.WebIndex.WiUI/AppShell.xaml b/src/WebExpress.WebIndex.WiUI/AppShell.xaml new file mode 100644 index 0000000..406429e --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/AppShell.xaml @@ -0,0 +1,20 @@ + + + + + + + + diff --git a/src/WebExpress.WebIndex.WiUI/AppShell.xaml.cs b/src/WebExpress.WebIndex.WiUI/AppShell.xaml.cs new file mode 100644 index 0000000..0331be5 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/AppShell.xaml.cs @@ -0,0 +1,10 @@ +ο»Ώnamespace WebExpress.WebIndex.WiUI +{ + public partial class AppShell : Shell + { + public AppShell() + { + InitializeComponent(); + } + } +} diff --git a/src/WebExpress.WebIndex.WiUI/Converters/FieldTypeConverter.cs b/src/WebExpress.WebIndex.WiUI/Converters/FieldTypeConverter.cs new file mode 100644 index 0000000..9a3ac64 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/Converters/FieldTypeConverter.cs @@ -0,0 +1,28 @@ +ο»Ώusing System.Text.Json; +using System.Text.Json.Serialization; +using WebExpress.WebIndex.WiUI.Model; + +namespace WebExpress.WebIndex.WiUI.Converters +{ + /// + /// Custom converter for field typ to string. + /// + internal class FieldTypeConverter : JsonConverter + { + /// + /// Converts a string to an int. + /// + public override FieldType Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return FieldTypeExtention.FromStringValue(reader.GetString()!); + } + + /// + /// Converts an int to a string. + /// + public override void Write(Utf8JsonWriter writer, FieldType value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToString()); + } + } +} diff --git a/src/WebExpress.WebIndex.WiUI/Converters/NullToBooleanConverter.cs b/src/WebExpress.WebIndex.WiUI/Converters/NullToBooleanConverter.cs new file mode 100644 index 0000000..abf61f9 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/Converters/NullToBooleanConverter.cs @@ -0,0 +1,38 @@ +ο»Ώusing System.Globalization; + +namespace WebExpress.WebIndex.WiUI.Converters +{ + /// + /// Converts a null value to false and any non-null value to true. + /// + public class NullToBooleanConverter : IValueConverter + { + /// + /// Converts a value to a boolean. Returns false if the value is null, otherwise returns true. + /// + /// The value to be converted. + /// The type of the binding target property. + /// Optional parameter to be used in the converter logic. + /// The culture to be used in the converter. + /// False if the value is null, otherwise true. + public object Convert(object? value, Type targetType, object? parameter, CultureInfo culture) + { + return value != null; + } + + /// + /// Converts a boolean value back to its original value. This method is not implemented and will throw an exception if called. + /// + /// The value to be converted back. + /// The type to convert to. + /// Optional parameter to be used in the converter logic. + /// The culture to be used in the converter. + /// Throws a NotImplementedException. + /// Thrown always as this method is not implemented. + public object ConvertBack(object? value, Type targetType, object? parameter, CultureInfo culture) + { + throw new NotImplementedException(); + } + } + +} diff --git a/src/WebExpress.WebIndex.WiUI/IndexManager.cs b/src/WebExpress.WebIndex.WiUI/IndexManager.cs new file mode 100644 index 0000000..2c8d796 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/IndexManager.cs @@ -0,0 +1,142 @@ +ο»Ώusing System.Globalization; +using WebExpress.WebIndex.Wql; + +namespace WebExpress.WebIndex.WiUI +{ + /// + /// Implementation of the index manager. + /// + public class IndexManager : WebIndex.IndexManager + { + /// + /// Initializes a new instance of the class. + /// + public IndexManager() + { + } + + /// + /// Registers a data type in the index. + /// + /// The data type. This must have the IIndexItem interface. + /// The culture. + /// The index type. + public void Create(Type dataType, CultureInfo culture, IndexType type = IndexType.Memory) + { + var genericMethod = typeof(IndexManager).GetMethod("Create", 1, [typeof(CultureInfo), typeof(IndexType)]); + var specificMethod = genericMethod?.MakeGenericMethod(dataType); + + specificMethod?.Invoke(this, [culture, type]); + } + + /// + /// Closes the index file of type T. + /// + /// The data type. This must have the IIndexItem interface. + public void Close(Type dataType) + { + var genericMethod = typeof(IndexManager).GetMethod("Close", 1, []); + var specificMethod = genericMethod?.MakeGenericMethod(dataType); + + specificMethod?.Invoke(this, []); + } + + /// + /// Removes all index documents of type. + /// + /// The data type. This must have the IIndexItem interface. + public void Drop(Type dataType) + { + var genericMethod = typeof(IndexManager).GetMethod("Drop", 1, []); + var specificMethod = genericMethod?.MakeGenericMethod(dataType); + + specificMethod?.Invoke(this, []); + } + + /// + /// Adds a item to the index. + /// + /// The data type. This must have the IIndexItem interface. + /// The data to be added to the index. + public void Insert(Type dataType, object item) + { + var genericMethod = typeof(IndexManager).GetMethods() + .Where(m => m.Name == "Insert" && m.IsGenericMethodDefinition) + .First(); + var specificMethod = genericMethod.MakeGenericMethod(dataType); + + specificMethod.Invoke(this, [item]); + } + + /// + /// Clear all data from index document. + /// + /// The data type. This must have the IIndexItem interface. + public void Clear(Type dataType) + { + var genericMethod = typeof(IndexManager).GetMethod("Clear", 1, []); + var specificMethod = genericMethod?.MakeGenericMethod(dataType); + + specificMethod?.Invoke(this, []); + } + + /// + /// Executes a wql statement. + /// + /// The data type. This must have the IIndexItem interface. + /// The data type. This must have the IIndexItem interface. + /// The wql statement. + /// The WQL statement. + public IWqlStatement Retrieve(Type dataType, string wql) + { + if (dataType == null) + { + return null; + } + + var genericMethod = typeof(IndexManager).GetMethod("Retrieve", 1, [typeof(string)]); + var specificMethod = genericMethod?.MakeGenericMethod(dataType); + + return specificMethod?.Invoke(this, [wql]) as IWqlStatement; + } + + /// + /// Counts the number of items of the index. + /// + /// The data type. This must have the IIndexItem interface. + /// The number of items. + public uint Count(Type dataType) + { + var genericMethod = typeof(IndexManager).GetMethod("Count", 1, []); + var specificMethod = genericMethod?.MakeGenericMethod(dataType); + + return (uint)specificMethod?.Invoke(this, [])!; + } + + /// + /// Returns all documents from the index. + /// + /// The data type. This must have the IIndexItem interface. + /// An enumeration of the documents. + public IEnumerable All(Type dataType) + { + var genericMethod = typeof(IndexManager).GetMethod("All", 1, []); + var specificMethod = genericMethod.MakeGenericMethod(dataType); + + return specificMethod.Invoke(this, []) as IEnumerable; + } + + /// + /// Returns an index type based on its type. + /// + /// The data type. This must have the IIndexItem interface. + /// The index type or null. + public IIndexDocument GetIndexDocument(Type dataType) + { + var genericMethod = typeof(IndexManager).GetMethod("GetIndexDocument", 1, []); + var specificMethod = genericMethod.MakeGenericMethod(dataType); + + return specificMethod.Invoke(this, []) as IIndexDocument; + } + } +} diff --git a/src/WebExpress.WebIndex.WiUI/MauiProgram.cs b/src/WebExpress.WebIndex.WiUI/MauiProgram.cs new file mode 100644 index 0000000..e074b13 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/MauiProgram.cs @@ -0,0 +1,27 @@ +ο»Ώusing CommunityToolkit.Maui; +using Microsoft.Extensions.Logging; + +namespace WebExpress.WebIndex.WiUI +{ + public static class MauiProgram + { + public static MauiApp CreateMauiApp() + { + var builder = MauiApp.CreateBuilder(); + builder + .UseMauiApp() + .UseMauiCommunityToolkit() + .ConfigureFonts(fonts => + { + fonts.AddFont("OpenSans-Regular.ttf", "OpenSansRegular"); + fonts.AddFont("OpenSans-Semibold.ttf", "OpenSansSemibold"); + }); + +#if DEBUG + builder.Logging.AddDebug(); +#endif + + return builder.Build(); + } + } +} diff --git a/src/WebExpress.WebIndex.WiUI/Model/Field.cs b/src/WebExpress.WebIndex.WiUI/Model/Field.cs new file mode 100644 index 0000000..ec00256 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/Model/Field.cs @@ -0,0 +1,28 @@ +ο»Ώnamespace WebExpress.WebIndex.WiUI.Model +{ + /// + /// Represents an index field. + /// + public class Field + { + /// + /// Returns or sets the name of the field. + /// + public string? Name { get; set; } + + /// + /// Returns or sets the type of the field. + /// + public FieldType Type { get; set; } + + /// + /// Returns or sets the ignore attribute. + /// + public bool Ignore { get; set; } + + /// + /// Returns or sets if the field is abstract. + /// + public bool Abstract { get; set; } + } +} diff --git a/src/WebExpress.WebIndex.WiUI/Model/FieldType.cs b/src/WebExpress.WebIndex.WiUI/Model/FieldType.cs new file mode 100644 index 0000000..b884744 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/Model/FieldType.cs @@ -0,0 +1,106 @@ +ο»Ώnamespace WebExpress.WebIndex.WiUI.Model +{ + /// + /// Represents the types of field that can be used. + /// + public enum FieldType + { + /// + /// Field type for plain text. + /// + Text, + + /// + /// Field type for boolean values. + /// + Bool, + + /// + /// Field type for integer values. + /// + Int, + + /// + /// Field type for double precision floating point numbers. + /// + Double, + + /// + /// Field type for date time values. + /// + DateTime, + + /// + /// Field type for guid values. + /// + Guid, + + /// + /// Field type for undefine value. + /// + Object + } + + /// + /// Provides extension methods for the FieldType enumeration. + /// + internal static class FieldTypeExtention + { + /// + /// Converts the FieldType value to a corresponding System.Type. + /// + /// The FieldType value to convert. + /// The System.Type that corresponds to the given FieldType value. + public static Type ToType(this FieldType type) + { + return type switch + { + FieldType.Text => typeof(string), + FieldType.Bool => typeof(bool), + FieldType.Int => typeof(int), + FieldType.Double => typeof(double), + FieldType.DateTime => typeof(DateTime), + FieldType.Guid => typeof(Guid), + _ => typeof(object) + }; + } + + /// + /// Converts the FieldType value to a corresponding string representation. + /// + /// The FieldType value to convert. + /// The string representation of the given FieldType value. + public static string ToString(this FieldType type) + { + return type switch + { + FieldType.Text => "Text", + FieldType.Bool => "Boolean", + FieldType.Int => "Integer", + FieldType.Double => "Double", + FieldType.DateTime => "DateTime", + FieldType.Guid => "Guid", + _ => "Object" + }; + } + + /// + /// Converts a string to a corresponding FieldType value. + /// + /// The string to convert. + /// The FieldType value that corresponds to the given string. + public static FieldType FromStringValue(string str) + { + return str switch + { + "String" => FieldType.Text, + "Boolean" => FieldType.Bool, + "Integer" => FieldType.Int, + "Double" => FieldType.Double, + "DateTime" => FieldType.DateTime, + "Guid" => FieldType.Guid, + _ => FieldType.Object + }; + } + } +} diff --git a/src/WebExpress.WebIndex.WiUI/Model/Index.cs b/src/WebExpress.WebIndex.WiUI/Model/Index.cs new file mode 100644 index 0000000..cb1fb18 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/Model/Index.cs @@ -0,0 +1,18 @@ +ο»Ώnamespace WebExpress.WebIndex.WiUI.Model +{ + /// + /// Represents an index file *.ws. + /// + public class Index + { + /// + /// Returns or sets the name of the field. + /// + public string? Name { get; set; } + + /// + /// Returns or sets the file name with its path. + /// + public string? FileNameWithPath { get; set; } + } +} diff --git a/src/WebExpress.WebIndex.WiUI/Model/MainViewModel.cs b/src/WebExpress.WebIndex.WiUI/Model/MainViewModel.cs new file mode 100644 index 0000000..bd5a392 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/Model/MainViewModel.cs @@ -0,0 +1,301 @@ +ο»Ώusing CommunityToolkit.Maui.Core.Extensions; +using System.Collections.ObjectModel; +using System.ComponentModel; +using System.Globalization; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Text.Json; +using WebExpress.WebIndex.Storage; +using WebExpress.WebIndex.WiUI.Converters; + +namespace WebExpress.WebIndex.WiUI.Model +{ + /// + /// Represents the main view model that implements the INotifyPropertyChanged interface. + /// + public class MainViewModel : INotifyPropertyChanged + { + private Project? _selectedProject = null; + private ObservableCollection _projects = []; + private Index? _selectedIndex = null; + private ObservableCollection _indexes = []; + private ObjectType? _selectedObjectType = null; + private Field? _selectedIndexField = null; + private Term? _selectedTerms = null; + private ObservableCollection _terms = []; + + /// + /// Occurs when a property value changes. + /// + public event PropertyChangedEventHandler? PropertyChanged; + + /// + /// Manages the indexing of project objects. + /// + public static IndexManager? IndexManager { get; private set; } + + /// + /// Returns or sets the selected project. + /// + /// + /// The selected project. + /// + public Project? SelectedProject + { + get => _selectedProject; + set + { + _selectedProject = value; OnPropertyChanged(); + + if (_selectedProject?.IndexPath != null && !Directory.Exists(_selectedProject?.IndexPath)) + { + Indexes = new ObservableCollection(); + + return; + } + + Indexes = Directory + .GetFiles(_selectedProject?.IndexPath ?? Environment.CurrentDirectory) + .Where(x => x.EndsWith(".ws")) + .Select(x => new Index() + { + Name = Path.GetFileNameWithoutExtension(x), + FileNameWithPath = x + }).ToObservableCollection(); + } + } + + /// + /// Returns or sets the collection of projects. + /// + public ObservableCollection Projects + { + get => _projects; + set { _projects = value; OnPropertyChanged(); } + } + + /// + /// Returns or sets the selected index. + /// + public Index? SelectedIndex + { + get => _selectedIndex; + set { _selectedIndex = value; OnPropertyChanged(); } + } + + /// + /// Returns or sets the collection of indexes. + /// + public ObservableCollection Indexes + { + get => _indexes; + set { _indexes = value; OnPropertyChanged(); } + } + + /// + /// Returns the selected field. + /// + /// + /// The selected field. + /// + public Field? SelectedField + { + get => _selectedIndexField; + set + { + _selectedIndexField = value; + _terms = GetIndexTerms(); + OnPropertyChanged(nameof(Terms)); + OnPropertyChanged(); + } + } + + /// + /// Returns the currently selected index file. + /// + /// + /// The name of the currently selected index file, or null if no index file is selected. + /// + public ObservableCollection? Fields + { + get { return _selectedObjectType?.Fields; } + } + + /// + /// Returns or sets the selected term. + /// + /// + /// The selected term. + /// + public Term? SelectedTerm + { + get { return _selectedTerms; } + set { _selectedTerms = value; OnPropertyChanged(); } + } + + /// + /// Returns the collection of terms. + /// + /// + /// The collection of terms. + /// + public ObservableCollection? Terms + { + get { return _terms; } + } + + /// + /// Initializes a new instance of the class. + /// + public MainViewModel() + { + Projects = ProjectService.LoadProjects(); + + SelectedProject = Projects.Where(x => x.IsSelected ?? false).FirstOrDefault(); + if (SelectedProject == null) + { + SelectedProject = Projects.FirstOrDefault(); + } + } + + /// + /// Opens the specified index file. + /// + /// The full path to the index file. + /// True if successful, otherwise fasle. + public bool CreateIndexFile(string indexFile) + { + _selectedObjectType = new ObjectType() { Name = indexFile }; + var indexFilewithPath = Path.Combine(SelectedProject?.IndexPath!, $"{indexFile}.ws"); + + var runtimeClass = _selectedObjectType.BuildRuntimeClass(); + var context = new IndexContext { IndexDirectory = indexFilewithPath }; + IndexManager = new IndexManager(); + + // use reflection to call the protected Initialization method + var method = typeof(IndexManager).GetMethod("Initialization", BindingFlags.Instance | BindingFlags.NonPublic); + method?.Invoke(IndexManager, [context]); + + IndexManager.Create(runtimeClass, CultureInfo.GetCultureInfo("en"), IndexType.Storage); + + OnPropertyChanged(nameof(Fields)); + + return true; + } + + /// + /// Opens the specified index file. + /// + /// The full path to the index file. + /// True if successful, otherwise fasle. + public bool OpenIndexFile(string? indexFile) + { + if (indexFile == null) + { + return false; + } + + CloseIndexFile(); + + var schema = File.ReadAllText(indexFile); + var options = new JsonSerializerOptions { Converters = { new FieldTypeConverter() } }; + _selectedObjectType = JsonSerializer.Deserialize(schema, options); + + var runtimeClass = _selectedObjectType?.BuildRuntimeClass(); + + var context = new IndexContext { IndexDirectory = Path.GetDirectoryName(indexFile) }; + IndexManager = new IndexManager(); + + // use reflection to call the protected Initialization method + var method = typeof(IndexManager).GetMethod("Initialization", BindingFlags.Instance | BindingFlags.NonPublic); + method?.Invoke(IndexManager, [context]); + + IndexManager.Create(runtimeClass!, CultureInfo.GetCultureInfo("en"), IndexType.Storage); + + OnPropertyChanged(nameof(Fields)); + + return true; + } + + /// + /// Opens the specified index field. + /// + /// The the index field. + /// True if successful, otherwise fasle. + public bool OpenIndexField(Field? indexField) + { + SelectedField = indexField; + + return true; + } + + /// + /// Close the current index file. + /// + /// True if successful, otherwise fasle. + public bool CloseIndexFile() + { + var runtimeClass = _selectedObjectType?.BuildRuntimeClass(); + IndexManager?.Close(runtimeClass!); + SelectedIndex = null; + + _selectedObjectType = null; + OnPropertyChanged(nameof(Fields)); + + return true; + } + + /// + /// Drop the current index file. + /// + /// True if successful, otherwise fasle. + public bool DropIndexFile() + { + var runtimeClass = _selectedObjectType?.BuildRuntimeClass(); + IndexManager?.Drop(runtimeClass!); + SelectedIndex = null; + _selectedObjectType = null; + + OnPropertyChanged(nameof(Fields)); + + return true; + } + + /// + /// Returns the index terms. + /// + /// The index terms + public ObservableCollection GetIndexTerms() + { + var runtimeClass = _selectedObjectType?.BuildRuntimeClass(); + var document = IndexManager?.GetIndexDocument(runtimeClass!); + var fieldProperty = runtimeClass?.GetProperty(_selectedIndexField?.Name!); + var fieldData = new IndexFieldData(fieldProperty); + var methodInfo = document?.GetType().GetMethod("GetReverseIndex"); + var reverseIndex = methodInfo?.Invoke(document, [fieldData]); + var termProperty = reverseIndex?.GetType().GetProperty("Term"); + var term = termProperty?.GetValue(reverseIndex) as IndexStorageSegmentTerm; + + return term?.Terms.Select(x => + ( + new Term() + { + Value = x.Item1, + Fequency = x.Item2.Fequency, + Height = x.Item2.Posting.Height, + Balance = x.Item2.Posting.Balance, + DocumentIDs = x.Item2.Posting.PreOrder.Select(y => y.DocumentID) + } + )).ToObservableCollection()!; + } + + /// + /// Raises the PropertyChanged event for the specified property. + /// + /// The name of the property that changed. + protected void OnPropertyChanged([CallerMemberName] string? propertyName = null) + { + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName)); + } + } +} diff --git a/src/WebExpress.WebIndex.WiUI/Model/ObjectType.cs b/src/WebExpress.WebIndex.WiUI/Model/ObjectType.cs new file mode 100644 index 0000000..f73da06 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/Model/ObjectType.cs @@ -0,0 +1,134 @@ +ο»Ώusing System.Collections.ObjectModel; +using System.ComponentModel; +using System.Reflection; +using System.Reflection.Emit; +using System.Runtime.CompilerServices; +using WebExpress.WebIndex.WebAttribute; + +namespace WebExpress.WebIndex.WiUI.Model +{ + /// + /// Represents an object within the WebExpress.WebIndex.Studio. + /// + public class ObjectType : INotifyPropertyChanged + { + private Dictionary _typeCache = []; + private ObservableCollection? _fields = []; + + /// + /// Occurs when a property value changes. + /// + public event PropertyChangedEventHandler? PropertyChanged; + + /// + /// Returns or sets the name of the object. + /// + public string? Name { get; set; } + + /// + /// Retuns a collection of fields associated with the object. + /// + public ObservableCollection? Fields + { + get { return _fields; } + set { _fields = value; OnPropertyChanged(); OnPropertyChanged(nameof(Fields)); } + } + + /// + /// Returns a collection of stored data objects. + /// + public ObservableCollection? All => new(MainViewModel.IndexManager?.All(BuildRuntimeClass())!); + + /// + /// Returns the number of items of the index. + /// + public uint? Count => MainViewModel.IndexManager?.Count(BuildRuntimeClass()); + + /// + /// Initializes a new instance of the class. + /// + public ObjectType() + { + + } + + /// + /// Dynamically builds a class based on the object's attributes. + /// + /// A Type representing the dynamically built class. + public Type BuildRuntimeClass() + { + if (_typeCache.TryGetValue(Name!, out Type? type)) + { + return type; + } + + var assemblyName = new AssemblyName("WebExpress.WebIndex.Wi.Model.Objects"); + var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run); + var moduleBuilder = assemblyBuilder.DefineDynamicModule($"Module_{Name}"); + + var typeBuilder = moduleBuilder.DefineType(Name!, TypeAttributes.Public, null, [typeof(IIndexItem)]); + + BildProperty(typeBuilder, "Id", typeof(Guid), true, true); + + foreach (var attribute in Fields!.Where(x => !x.Name!.Equals("id", StringComparison.OrdinalIgnoreCase))) + { + BildProperty(typeBuilder, attribute.Name!, attribute.Type.ToType(), attribute.Ignore, attribute.Abstract); + } + + var runtimeClass = typeBuilder.CreateType(); + _typeCache.Add(Name!, runtimeClass); + + return runtimeClass; + } + + /// + /// Builds a property for the dynamically created class. + /// + /// The builder for the class type. + /// The name of the property. + /// The type of the property. + /// Indicates whether the property should be ignored by the index. + /// Indicates whether the property should be virtual. + /// A PropertyBuilder for the created property. + private PropertyBuilder BildProperty(TypeBuilder typeBuilder, string name, Type type, bool indexIgnore, bool @virtual) + { + var fieldBuilder = typeBuilder.DefineField($"_{name.ToLower()}", type, FieldAttributes.Private); + var propertyBuilder = typeBuilder.DefineProperty(name, PropertyAttributes.HasDefault, type, Type.EmptyTypes); + + if (indexIgnore) + { + var attrCtorParams = Array.Empty(); + var attrCtorInfo = typeof(IndexIgnoreAttribute).GetConstructor(attrCtorParams); + var attrBuilder = new CustomAttributeBuilder(attrCtorInfo!, []); + propertyBuilder.SetCustomAttribute(attrBuilder); + } + + var getMethodBuilder = typeBuilder.DefineMethod($"get_{name}", @virtual ? MethodAttributes.Public | MethodAttributes.Virtual : MethodAttributes.Public, type, Type.EmptyTypes); + var getIlGenerator = getMethodBuilder.GetILGenerator(); + getIlGenerator.Emit(OpCodes.Ldarg_0); + getIlGenerator.Emit(OpCodes.Ldfld, fieldBuilder); + getIlGenerator.Emit(OpCodes.Ret); + propertyBuilder.SetGetMethod(getMethodBuilder); + + var setMethodBuilder = typeBuilder.DefineMethod($"set_{name}", @virtual ? MethodAttributes.Public | MethodAttributes.Virtual : MethodAttributes.Public, null, [type]); + var setIlGenerator = setMethodBuilder.GetILGenerator(); + setIlGenerator.Emit(OpCodes.Ldarg_0); + setIlGenerator.Emit(OpCodes.Ldarg_1); + setIlGenerator.Emit(OpCodes.Stfld, fieldBuilder); + setIlGenerator.Emit(OpCodes.Ret); + propertyBuilder.SetSetMethod(setMethodBuilder); + + return propertyBuilder; + } + + /// + /// Raises the PropertyChanged event for the specified property. + /// + /// The name of the property that changed. + private void OnPropertyChanged([CallerMemberName] string? propertyName = null) + { + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName)); + } + } +} diff --git a/src/WebExpress.WebIndex.WiUI/Model/Project.cs b/src/WebExpress.WebIndex.WiUI/Model/Project.cs new file mode 100644 index 0000000..0163b4c --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/Model/Project.cs @@ -0,0 +1,60 @@ +ο»Ώusing System.ComponentModel; +using System.Runtime.CompilerServices; + +namespace WebExpress.WebIndex.WiUI.Model +{ + /// + /// Represents an project. + /// + public class Project : INotifyPropertyChanged + { + private string? _projectName; + private bool? _isSelected; + + /// + /// Occurs when a property value changes. + /// + public event PropertyChangedEventHandler? PropertyChanged; + + /// + /// Returns or sets the name of the project. + /// + public string? ProjectName + { + get => _projectName; + set { _projectName = value; OnPropertyChanged(); } + } + + /// + /// Returns or sets the selected project. + /// + public bool? IsSelected + { + get => _isSelected; + set { _isSelected = value; OnPropertyChanged(); OnPropertyChanged(nameof(TextColor)); } + } + + /// + /// Returns the color based on the selection status of the project. + /// Blue if the project is selected, otherwise returns black. + /// + public Color? TextColor + { + get => _isSelected.HasValue && _isSelected.Value ? Color.FromRgb(20, 20, 255) : Color.FromRgb(0, 0, 0); + } + + /// + /// Returns or sets the index path. + /// + public string? IndexPath { get; set; } + + /// + /// Raises the PropertyChanged event for the specified property. + /// + /// The name of the property that changed. This is optional and will be automatically provided by the compiler if not specified. + protected void OnPropertyChanged([CallerMemberName] string? propertyName = null) + { + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName)); + } + } +} diff --git a/src/WebExpress.WebIndex.WiUI/Model/ProjectService.cs b/src/WebExpress.WebIndex.WiUI/Model/ProjectService.cs new file mode 100644 index 0000000..a4ec7b5 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/Model/ProjectService.cs @@ -0,0 +1,60 @@ +ο»Ώusing System.Collections.ObjectModel; +using System.Xml.Serialization; + +namespace WebExpress.WebIndex.WiUI.Model +{ + /// + /// Provides services for saving and loading projects. + /// + public class ProjectService + { + private static readonly string _appFolderName = "WebExpress.WebIndex.WiUI"; + private static readonly string _projectsFileName = "projects.xml"; + + /// + /// Returns the application data path for storing project files. + /// + /// The path to the application data directory. + private static string GetAppDataPath() + { + var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData); + var appFolder = Path.Combine(appDataPath, _appFolderName); + Directory.CreateDirectory(appFolder); + + return appFolder; + } + + /// + /// Saves the collection of projects to an XML file in the application data directory. + /// + /// The collection of projects to save. + public static void SaveProjects(ObservableCollection projects) + { + var filePath = Path.Combine(GetAppDataPath(), _projectsFileName); + + var serializer = new XmlSerializer(typeof(ObservableCollection)); + using var writer = new StreamWriter(filePath); + + serializer.Serialize(writer, projects); + } + + /// + /// Loads the collection of projects from an XML file in the application data directory. + /// + /// The collection of projects loaded from the XML file. + public static ObservableCollection LoadProjects() + { + var filePath = Path.Combine(GetAppDataPath(), _projectsFileName); + if (!File.Exists(filePath)) + { + return []; + } + + var serializer = new XmlSerializer(typeof(ObservableCollection)); + using StreamReader reader = new StreamReader(filePath); + var result = serializer.Deserialize(reader) as ObservableCollection; + + return result ?? []; + } + } +} diff --git a/src/WebExpress.WebIndex.WiUI/Model/ProjectViewModel.cs b/src/WebExpress.WebIndex.WiUI/Model/ProjectViewModel.cs new file mode 100644 index 0000000..24f5512 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/Model/ProjectViewModel.cs @@ -0,0 +1,116 @@ +ο»Ώusing System.Collections.ObjectModel; +using System.ComponentModel; +using System.Runtime.CompilerServices; +using System.Windows.Input; + +namespace WebExpress.WebIndex.WiUI.Model +{ + /// + /// ViewModel for loading projects. + /// + public class ProjectViewModel : INotifyPropertyChanged + { + private Project? _selectedProject = null; + + /// + /// Occurs when a property value changes. + /// + public event PropertyChangedEventHandler? PropertyChanged; + + /// + /// Returns the command to add a new project. + /// + public ICommand AddProjectCommand { get; } + + /// + /// Returns the command to delete an existing project. + /// + public ICommand DeleteProjectCommand { get; } + + /// + /// Returns the command to save the current project. + /// + public ICommand SaveProjectCommand { get; } + + /// + /// Returns or sets the collection of projects. + /// + public ObservableCollection Projects { get; set; } + + /// + /// Returns or sets the selected project. + /// + public Project? SelectedProject + { + get => _selectedProject; + set + { + _selectedProject = value; + + Projects.ToList().ForEach(p => p.IsSelected = false); + + if (_selectedProject != null) + { + _selectedProject.IsSelected = true; + } + + OnPropertyChanged(); + } + } + + /// + /// Initializes a new instance of the class. + /// Loads the projects and initializes the load project command. + /// + public ProjectViewModel() + { + Projects = ProjectService.LoadProjects(); + AddProjectCommand = new Command(OnAddProject); + DeleteProjectCommand = new Command(OnDeleteProject, () => SelectedProject != null); + SaveProjectCommand = new Command(OnSaveProject, () => SelectedProject != null); + } + + /// + /// Adds a new project to the collection and saves the updated collection. + /// + private void OnAddProject() + { + var newProject = new Project { ProjectName = "New Project" }; + Projects.Add(newProject); + SelectedProject = newProject; + ProjectService.SaveProjects(Projects); + } + + /// + /// Deletes the selected project from the collection and saves the updated collection. + /// + private void OnDeleteProject() + { + if (SelectedProject != null) + { + Projects.Remove(SelectedProject); + ProjectService.SaveProjects(Projects); + } + } + + /// + /// Saves the current project to the project collection. + /// + private void OnSaveProject() + { + if (SelectedProject != null) + { + ProjectService.SaveProjects(Projects); + } + } + + /// + /// Raises the PropertyChanged event for the specified property. + /// + /// The name of the property that changed. + protected void OnPropertyChanged([CallerMemberName] string? propertyName = null) + { + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName)); + } + } +} \ No newline at end of file diff --git a/src/WebExpress.WebIndex.WiUI/Model/Term.cs b/src/WebExpress.WebIndex.WiUI/Model/Term.cs new file mode 100644 index 0000000..a7f651c --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/Model/Term.cs @@ -0,0 +1,38 @@ +ο»Ώnamespace WebExpress.WebIndex.WiUI.Model +{ + /// + /// Represents an index term. + /// + public class Term + { + /// + /// Returns or sets the collection of document IDs associated with the term. + /// + public IEnumerable DocumentIDs { get; set; } = []; + + /// + /// Returns a comma-separated string of document IDs associated with the term. + /// + public uint Documents => (uint)DocumentIDs.Count(); + + /// + /// Returns or sets the term. + /// + public string? Value { get; set; } + + /// + /// Returns or sets the frequency of the term in the documents. + /// + public uint Fequency { get; set; } + + /// + /// Returns or sets the height of the term tree. + /// + public uint Height { get; set; } + + /// + /// Returns or sets the balance factor of the term tree. + /// + public uint Balance { get; set; } + } +} diff --git a/src/WebExpress.WebIndex.WiUI/Pages/MainPage.xaml b/src/WebExpress.WebIndex.WiUI/Pages/MainPage.xaml new file mode 100644 index 0000000..e8c1764 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/Pages/MainPage.xaml @@ -0,0 +1,204 @@ +ο»Ώ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/WebExpress.WebIndex.WiUI/Pages/MainPage.xaml.cs b/src/WebExpress.WebIndex.WiUI/Pages/MainPage.xaml.cs new file mode 100644 index 0000000..ce83807 --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/Pages/MainPage.xaml.cs @@ -0,0 +1,99 @@ +ο»Ώusing WebExpress.WebIndex.WiUI.Model; + +namespace WebExpress.WebIndex.WiUI.Pages +{ + /// + /// Represents the main page of the application. + /// Inherits from . + /// + public partial class MainPage : ContentPage + { + /// + /// Initializes a new instance of the class. + /// Sets the BindingContext to a new instance of . + /// + public MainPage() + { + InitializeComponent(); + BindingContext = new MainViewModel(); + } + + /// + /// Handles the event when an index is selected. + /// Displays an alert with the selected index. + /// + /// The source of the event. + /// The event data containing the selected item. + private void OnIndexSelected(object sender, SelectedItemChangedEventArgs e) + { + if (e.SelectedItem == null) + { + return; + } + + var selectedIndex = e.SelectedItem as Model.Index; + var context = BindingContext as MainViewModel; + + var fileNameWithPath = selectedIndex?.FileNameWithPath; + + if (File.Exists(fileNameWithPath)) + { + context?.OpenIndexFile(fileNameWithPath); + } + } + + /// + /// Handles the event when a field is selected. + /// Displays an alert with the selected field. + /// + /// The source of the event. + /// The event data containing the selected item. + private void OnFieldSelected(object sender, SelectedItemChangedEventArgs e) + { + if (e.SelectedItem == null) + { + return; + } + + var selectedIndex = e.SelectedItem as Model.Field; + var context = BindingContext as MainViewModel; + + context?.OpenIndexField(selectedIndex); + } + + + /// + /// Handles the event when a term is selected. + /// Displays an alert with the selected term. + /// + /// The source of the event. + /// The event data containing the selected item. + private void OnTermSelected(object sender, SelectedItemChangedEventArgs e) + { + if (e.SelectedItem == null) + { + return; + } + + var selectedTerm = e.SelectedItem as Model.Term; + var context = BindingContext as MainViewModel; + } + + /// + /// Handles the event when a term is added to the clipboard. + /// Copies the selected term's value to the clipboard and displays an alert. + /// + /// The source of the event. + /// The event data. + private async void OnAddClipboard(object sender, EventArgs e) + { + var button = sender as ImageButton; + + if (button?.BindingContext is Model.Term selectedTerm) + { + await Clipboard.Default.SetTextAsync(selectedTerm.Value); + await DisplayAlert("Copied", $"'{selectedTerm.Value}' copied to clipboard", "OK"); + } + } + } +} diff --git a/src/WebExpress.WebIndex.WiUI/Pages/ProjectPage.xaml b/src/WebExpress.WebIndex.WiUI/Pages/ProjectPage.xaml new file mode 100644 index 0000000..ef4962f --- /dev/null +++ b/src/WebExpress.WebIndex.WiUI/Pages/ProjectPage.xaml @@ -0,0 +1,72 @@ + + + + + + + + + + +