From cfa8867ca1a276f33a1799aec7d45362acfbcf46 Mon Sep 17 00:00:00 2001 From: Zaine Date: Tue, 2 Jun 2026 12:53:00 +0100 Subject: [PATCH] changes --- .obsidian/app.json | 3 +- .obsidian/workspace.json | 68 +- Career/20250723200800-postgres.md | 3 + Career/20250727122406-database_moc.md | 3 + Career/20250804201706-big_o_complexity.md | 94 +++ .../20251223215636-design_patterns_notes.md | 162 ++++ Career/20251229180039-airflow.md | 20 + Career/20251229180156-airflow_tasks.md | 9 + Career/20251229180249-airflow_dags.md | 1 + Career/20260101011126-xor.md | 30 + Career/20260101012549-solid_principles.md | 15 + ...0260101012731-solid_principles_examples.md | 1 + Career/20260115152626-apim.md | 55 ++ Career/20260121173318-api_architecture.md | 549 ++++++++++++++ Career/20260305102210-cicd_example.md | 546 +++++++++++++ Career/20260305111610-cicd_summary.md | 62 ++ ...305130457-database_perms_roles_accounts.md | 238 ++++++ Career/20260305132058-restful_api.md | 296 ++++++++ ...0260327114708-asp_net_core_6_apis_notes.md | 1 + ...260328215014-software_dev_methodologies.md | 337 ++++++++ Career/20260407102129-mvp_mvt.md | 15 + Career/20260410115348-microlise_ess.md | 103 +++ Career/20260410121527-ess_esp_index.md | 58 ++ Career/20260410121719-esp_applications.md | 90 +++ Career/20260410121927-esp_pipeline.md | 150 ++++ Career/20260410122116-ess_scripts.md | 270 +++++++ Career/20260410122303-ess_manifest.md | 118 +++ Career/20260410122425-ess_database.md | 126 +++ Career/20260410122604-ess_deploy_stages.md | 161 ++++ Career/20260410122754-esp_known_issues.md | 161 ++++ Career/20260410122935-esp_open_questions.md | 171 +++++ Career/20260410123131-esp_glossary.md | 197 +++++ Career/20260413120825-windows_services.md | 75 ++ Career/20260413154901-dll.md | 1 + .../20260505121135-seb_search_improvements.md | 180 +++++ .../20260512121806-data_camp_ai_training.md | 161 ++++ Career/20260513115140-powershell_moc.md | 8 + ...20260513142352-aritificial_intelligence.md | 15 + ...0260527152602-microlise_xss_rom_pentest.md | 429 +++++++++++ Career/Career MOC.md | 86 +++ Career/Concepts.md | 133 ++++ Career/Java Portswrigger Test.md | 717 ++++++++++++++++++ Career/Job applications.md | 275 +++++++ Career/Microlise MOC.md | 9 + .../CV's/Cover Letter/Santec/Santec.pdf.md | 0 45 files changed, 6172 insertions(+), 30 deletions(-) create mode 100644 Career/20250723200800-postgres.md create mode 100644 Career/20250727122406-database_moc.md create mode 100644 Career/20250804201706-big_o_complexity.md create mode 100644 Career/20251223215636-design_patterns_notes.md create mode 100644 Career/20251229180039-airflow.md create mode 100644 Career/20251229180156-airflow_tasks.md create mode 100644 Career/20251229180249-airflow_dags.md create mode 100644 Career/20260101011126-xor.md create mode 100644 Career/20260101012549-solid_principles.md create mode 100644 Career/20260101012731-solid_principles_examples.md create mode 100644 Career/20260115152626-apim.md create mode 100644 Career/20260121173318-api_architecture.md create mode 100644 Career/20260305102210-cicd_example.md create mode 100644 Career/20260305111610-cicd_summary.md create mode 100644 Career/20260305130457-database_perms_roles_accounts.md create mode 100644 Career/20260305132058-restful_api.md create mode 100644 Career/20260327114708-asp_net_core_6_apis_notes.md create mode 100644 Career/20260328215014-software_dev_methodologies.md create mode 100644 Career/20260407102129-mvp_mvt.md create mode 100644 Career/20260410115348-microlise_ess.md create mode 100644 Career/20260410121527-ess_esp_index.md create mode 100644 Career/20260410121719-esp_applications.md create mode 100644 Career/20260410121927-esp_pipeline.md create mode 100644 Career/20260410122116-ess_scripts.md create mode 100644 Career/20260410122303-ess_manifest.md create mode 100644 Career/20260410122425-ess_database.md create mode 100644 Career/20260410122604-ess_deploy_stages.md create mode 100644 Career/20260410122754-esp_known_issues.md create mode 100644 Career/20260410122935-esp_open_questions.md create mode 100644 Career/20260410123131-esp_glossary.md create mode 100644 Career/20260413120825-windows_services.md create mode 100644 Career/20260413154901-dll.md create mode 100644 Career/20260505121135-seb_search_improvements.md create mode 100644 Career/20260512121806-data_camp_ai_training.md create mode 100644 Career/20260513115140-powershell_moc.md create mode 100644 Career/20260513142352-aritificial_intelligence.md create mode 100644 Career/20260527152602-microlise_xss_rom_pentest.md create mode 100644 Career/Career MOC.md create mode 100644 Career/Concepts.md create mode 100644 Career/Java Portswrigger Test.md create mode 100644 Career/Job applications.md create mode 100644 Career/Microlise MOC.md create mode 100644 home/zaine/master-folder/CV's/Cover Letter/Santec/Santec.pdf.md diff --git a/.obsidian/app.json b/.obsidian/app.json index 9d85999..da6c968 100644 --- a/.obsidian/app.json +++ b/.obsidian/app.json @@ -1,3 +1,4 @@ { - "attachmentFolderPath": "./Attachments" + "attachmentFolderPath": "./Attachments", + "alwaysUpdateLinks": true } \ No newline at end of file diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index 347746d..2abde9d 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -4,21 +4,21 @@ "type": "split", "children": [ { - "id": "29bb1f246804879b", + "id": "1a0df7eb82aa0417", "type": "tabs", "children": [ { - "id": "60b312a6ea2225f6", + "id": "0aa065275eb81420", "type": "leaf", "state": { "type": "markdown", "state": { - "file": "Index.md", + "file": "Career/Career MOC.md", "mode": "source", "source": false }, "icon": "lucide-file", - "title": "Index" + "title": "Career MOC" } } ] @@ -180,36 +180,46 @@ "bases:Create new base": false } }, - "active": "60b312a6ea2225f6", + "active": "0aa065275eb81420", "lastOpenFiles": [ + "Career/Microlise MOC.md", + "Career/Concepts.md", + "Career/Java Portswrigger Test.md", + "Career/Career MOC.md", + "Career/Job applications.md", + "home/zaine/master-folder/CV's/Cover Letter/Santec/Santec.pdf.md", + "home/zaine/master-folder/CV's/Cover Letter/Santec", + "home/zaine/master-folder/CV's/Cover Letter", + "home/zaine/master-folder/CV's", + "home/zaine/master-folder", + "home/zaine", + "home", + "Index.md", + "Career/20260527152602-microlise_xss_rom_pentest.md", + "Career/20260513142352-aritificial_intelligence.md", + "Career/20260513115140-powershell_moc.md", + "Career/20260512121806-data_camp_ai_training.md", + "Career/20260505121135-seb_search_improvements.md", + "Career/20260413154901-dll.md", + "Career/20260413120825-windows_services.md", + "Career/20260410123131-esp_glossary.md", + "Career/20260410122935-esp_open_questions.md", + "Career/20260410122754-esp_known_issues.md", + "Career/20260410122604-ess_deploy_stages.md", + "Career/20260410122425-ess_database.md", + "Career/20260410122303-ess_manifest.md", + "Career/20260410122116-ess_scripts.md", + "Career/20260410121927-esp_pipeline.md", + "Career/20260410121719-esp_applications.md", + "Career/20260410121527-ess_esp_index.md", + "Career/20260410115348-microlise_ess.md", + "Career/20260407102129-mvp_mvt.md", + "Career", "Attachments/anime-kuroko-no-basket-tetsuya-basketball-play-17f4blaudede9nw2.gif", "Attachments", "anime-kuroko-no-basket-tetsuya-basketball-play-17f4blaudede9nw2.gif", - "Daily Notes.md", - "Dailies/2026-05-05.md", - "Dailies/2026-05-13.md", - "Dailies/2026-04-21.md", - "Dailies/2026-05-12.md", - "Dailies/2026-04-14.md", - "Index.md", "systemd/quartz-notes.service", "systemd", - "Dailies/2026-06-02.md", - "Untitled.canvas", - "2026-06-02.md", - "Dailies/2026-05-28.md", - "Dailies/2026-06-01.md", - "Dailies/2026-05-29.md", - "Dailies/2026-05-22.md", - "Dailies/2026-05-21.md", - "Dailies/2026-05-20.md", - "Dailies/2026-05-06.md", - "Dailies/2026-05-01.md", - "Dailies/2026-04-30.md", - "Dailies/2026-04-29.md", - "Dailies/2026-04-28.md", - "Dailies/2026-04-20.md", - "Dailies/2026-04-18.md", - "Dailies" + "Untitled.canvas" ] } \ No newline at end of file diff --git a/Career/20250723200800-postgres.md b/Career/20250723200800-postgres.md new file mode 100644 index 0000000..3025924 --- /dev/null +++ b/Career/20250723200800-postgres.md @@ -0,0 +1,3 @@ +``` bash +psql -h 82.18.104.48 -p 5432 -U zaine -d org_web +``` diff --git a/Career/20250727122406-database_moc.md b/Career/20250727122406-database_moc.md new file mode 100644 index 0000000..e7c5abb --- /dev/null +++ b/Career/20250727122406-database_moc.md @@ -0,0 +1,3 @@ + - [Postgres](id:939e301b-6463-46a8-b57e-0af606e7e7ef) + + - [Airflow](id:c6cf8f7a-778f-4e83-b607-753ef8dbb3f1) diff --git a/Career/20250804201706-big_o_complexity.md b/Career/20250804201706-big_o_complexity.md new file mode 100644 index 0000000..76555f3 --- /dev/null +++ b/Career/20250804201706-big_o_complexity.md @@ -0,0 +1,94 @@ +# Big (O) - Time and Space Complexity + +## Intro + +Time Complexity: Describes the amount of time necessary to execute an algorithm + +Space Complexity: Describes the amount of memory or space utilized by an algorithm/program + +Both - asymptotically + +## Technical Definition of Big O + +is a mathematical notation that describes the limiting behaviour of a function when the arguments tend towards a particular value or infinity. Why do we need it? it helps us understand how the performance of an algorithm changes as the size of the input grows, providing a simple way to compare and analyse different algorithms' efficiency. + +Improvement in time complexity is often more important as memory is cheap and readily available + +In Big O, there are six major types of complexities (time and space): + + - Constant: O(1) + + - Linear time: O(n) + + - Logarithmic time: O(n log n) + + - Quadratic time: O(n2) + + - Exponential time: O(2n) + + - Factorial time: O(n\!) + +## Big O - linear example + +Suppose we are given a problem where we have a list of \`N\` numbers of unknown length. We are asked to use code to find and return "True" if the number 2 is in the list and "False" otherwise. Our solution could be to go through every position in the list and check if the number at that position is equal to 2. + +\[3, 10, 2, 7\] + +``` python +for number in list: + if number == 2: + return True + else: + continue +return False +``` + +This would take N time, we need to check every number in the list once, making this solution O(N) - linear time. This looks at the worst case scenarion, if 2 was at the start of the list we know it would take a constant time, however if it's at the end then it would take N time. + +![](../assets/Big-O-Notation-3130482830.png) + +In the graph above focus on the tail end of the graphs because Big O is concerned with "as the input size grows what happens to the speed of the operations". + +| | | | | | +| ---------- | ------------ | -------------------------------------------------------------------------- | ---------------------------------- | -------------------- | +| Complexity | Name | Description | Common Use Cases | Performance at Scale | +| O(1) | Constant | Runtime unaffected by input size | Hash tables, array access | Excellent | +| O(log n) | Logarithmic | Runtime increases slowly (typically halved at each step) | Binary search, balanced trees | Very good | +| O(n) | Linear | Runtime scales linearly (proportional) | Linear search, array traversal | Good | +| O(n log n) | Linearithmic | Between linear and quadratic (often seen in divide and conquer algorithms) | Efficient sorting algorithms | Fair | +| O(n²) | Quadratic | Runtime squares with input size | Nested loops, simple sorting | Poor | +| O(2ⁿ) | Exponential | Runtime doubles with each input | Recursive solutions, combinatorics | Very poor | +| O(n\!) | Factorial | Runtime grows by factorial (extremely slow) | Permutations, traveling salesman | Terrible | + +## Summation of complexities: + +When you have multiple operations in an algorithm that each have a linear time complexity O(n), and these operations are sequential (not nested), the overall time complexity of the algorithm remains linear, O(n). + +Here's how it works: + +1. Summing Linear Operations: If your algorithm involves several separate linear operations, such as: + +• First iterating over an array of n elements, +• Then, in a separate loop, iterating over the same or another array of n elements, +• And perhaps another loop doing the same, +each operation has a complexity of O(n). If you sum these, the resulting complexity for these sequential operations is O(n) + O(n) + O(n), and so on. + +1. Simplification: According to Big O notation rules, when you add complexities of the same order, the overall complexity is dominated by the term that grows fastest as n increases. For linear operations, O(n) + O(n) + O(n) simplifies to O(n) because the growth rate in terms of the largest input size n doesn't change-it remains linear. + +Linear time complexity, denoted as O(n), means that the time required to complete the execution of an algorithm increases linearly with the increase in the size of the input data. In essence, if you double the size of the input, you double the time it takes to process it. + +Suppose you have a task to sum the number 5, n times. The number of operations (in this case, additions) you perform directly corresponds to n. For instance: + + - If n = 1, you perform the operation 1 time: 5. + - If n = 2, you perform the operation 2 times: 5+5. + - If n = 3, you perform the operation 3 times: 5+5+5. + - And so on… + +In each of these cases, the number of addition operations you perform is exactly equal to n. The computational cost grows directly with n, which is the very definition of linear time complexity. Here's a breakdown: + + - When n = 1, the number of operations is 1. + - When n = 10, the number of operations is 10. + - When n = 100, the number of operations is 100. + - When n = 1000, the number of operations is 1000. + +In general, the total time taken for this task can be described as a function T(n) = n, where T(n) represents the total time or total number of operations, and n is the number of times you need to add 5. This function is a straight line when plotted against n, hence it is classified under linear time complexity O(n). diff --git a/Career/20251223215636-design_patterns_notes.md b/Career/20251223215636-design_patterns_notes.md new file mode 100644 index 0000000..321173d --- /dev/null +++ b/Career/20251223215636-design_patterns_notes.md @@ -0,0 +1,162 @@ +**WIP** + +# Summary of the major patterns: + +## The Strategy Pattern + +Strategy is a behavioral design pattern that lets you define a family of algorithms, put each of them into a separate class, and make their objects interchangeable. + +![](./assets/design-patterns/structure-pattern.png) + +## Observer pattern + +Observer is a behavioral design pattern that lets you define a subscription mechanism to notify multiple objects about any events that happen to the object they’re observing. + +![](./assets/design-patterns/observer-pattern.png) + +## Decorator pattern + +Decorator is a structural design pattern that lets you attach new behaviors to objects by placing these objects inside special wrapper objects that contain the behaviors. + +The problem: + +![](./assets/design-patterns/decorator-problem.png) + +![](./assets/design-patterns/decorator-pattern.png) + +## Factory Pattern + +Factory Method is a creational design pattern that provides an interface for creating objects in a superclass, but allows subclasses to alter the type of objects that will be created. + +![](./assets/design-patterns/factory-pattern.png) + +## Singleton Pattern + +Singleton is a creational design pattern that lets you ensure that a class has only one instance, while providing a global access point to this instance. + +![](./assets/design-patterns/singleton-pattern.png) + +## Command Pattern + +Command is a behavioral design pattern that turns a request into a stand-alone object that contains all information about the request. This transformation lets you pass requests as a method arguments, delay or queue a request’s execution, and support undoable operations. + +![](./assets/design-patterns/command-pattern.png) + +## Adapter Pattern + +Adapter is a structural design pattern that allows objects with incompatible interfaces to collaborate. + +![](./assets/design-patterns/adapter-pattern.png) + +Another approach: + +![](./assets/design-patterns/adapter-pattern-2.png) + +## Facade Pattern + +Facade is a structural design pattern that provides a simplified interface to a library, a framework, or any other complex set of classes. + +![](./assets/design-patterns/facade-pattern.png) + +## Template Method Pattern + +Template Method is a behavioral design pattern that defines the skeleton of an algorithm in the superclass but lets subclasses override specific steps of the algorithm without changing its structure. + +![](./assets/design-patterns/template-method-pattern.png) + +## Iterator and Composite Pattern + +Iterator is a behavioral design pattern that lets you traverse elements of a collection without exposing its underlying representation (list, stack, tree, etc.). + +![](./assets/design-patterns/iterator-pattern.png) + +Composite is a structural design pattern that lets you compose objects into tree structures and then work with these structures as if they were individual objects. + +![](./assets/design-patterns/composite-pattern.png) + +## State Pattern + +State is a behavioral design pattern that lets an object alter its behavior when its internal state changes. It appears as if the object changed its class. + +![](./assets/design-patterns/state-pattern.png) + +## Proxy Pattern + +Proxy is a structural design pattern that lets you provide a substitute or placeholder for another object. A proxy controls access to the original object, allowing you to perform something either before or after the request gets through to the original object. + +![](./assets/design-patterns/proxy-pattern.png) + +# Design patterns. What are they? + +They are reusable solutions to common problems in software design. They help to make code more flexible, maintainable, and scalable. + +Patterns allow you to say more with less. When you use a pattern in a description, other developers quickly know precisely the design you have in mind. + +## OO concepts: + +### Abstraction: + +Focuses on essential features while hiding unnecessary internal details, allowing developers to work with high‑level concepts instead of implementation complexity. + +Example: A Car class exposes start() and stop() methods without revealing how the engine ignition system works. + +### Inheritance: + +Enables one class to derive properties and behaviours from another, promoting code reuse and creating natural parent–child hierarchies. + +Example: A Dog class inherits from an Animal class, automatically gaining attributes like age and methods like eat(). + +### Encapsulation: + +Protects an object’s internal state by restricting direct access to its data and exposing controlled interfaces for interaction. + +Example: A BankAccount class keeps its balance private and provides deposit() and withdraw() methods to modify it safely. + +### Polymorphism: + +Allows different objects to respond to the same interface or method call in their own unique ways, enabling flexible and extensible system design. + +Example: Calling makeSound() on an Animal reference triggers bark() for a Dog and meow() for a Cat. + +## OO principles: + + - Encapsulate what varies + + - Favor composition over inheritance + + - Program to interfaces, not implementations + + - Strive for loosely coupled designs between objects that interact + +## OO Patterns + +You have: + + - **behavioural** patterns + - **creational** patterns + - **structural** patterns + +### Behavioural patterns + +1. Structural: + + The Strategy Pattern defines a family of algorithms, encapsulates each one, and makes them interchangeable. Strategy lets the algorithm vary independently from clients that use it. + +2. Observer: + + The Observer Pattern defines a one-to-many dependency between objects so that when + one object changes state, all of its dependents are notified and updated automatically. + + Subjects, or as we also know them, Observables, update Observers using a common + interface + + Observers are loosely coupled in that the Observable knows nothing about them, + other than that they implement the Observer interface. + + You can push or pull data from the Observable when using the pattern (pull is + considered more “correct”). + + Don’t depend on a specific order of notification for your Observers. + + Java has several implementations of the Observer Pattern, including the general + purpose java.util.Observable diff --git a/Career/20251229180039-airflow.md b/Career/20251229180039-airflow.md new file mode 100644 index 0000000..343de10 --- /dev/null +++ b/Career/20251229180039-airflow.md @@ -0,0 +1,20 @@ +# xcom + +XComs stands for cross-communications, and as the name suggests, it allows [Airflow Tasks](id:1ea42d73-3cd1-4cac-a6ee-028de21d08d5) to talk to each other (because by default, tasks are entirely isolated and can run on seperate machines). + +See [this](https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/xcoms.html) + +# parquets + +**Key Takeaways** + + - What is Parquet: An open-source columnar storage file format for efficient analytics. + - Core Benefits: Superior compression, faster query performance (via column pruning and predicate pushdown), and schema evolution support. + - Common Alternatives: Compared to row-based (CSV, Avro), columnar (ORC), and table formats (Iceberg, Delta Lake). + - DuckDB & MotherDuck: Parquet integrates seamlessly with DuckDB for high-performance SQL queries directly on Parquet files. + +**What is Parquet?** + +Parquet is a columnar storage file format. When data engineers ask 'what is a Parquet file?', the simple answer is that it's a file that stores data in columns, not rows. This Parquet data format is designed for efficient data processing, particularly in the context of big data applications. Developed as part of the Apache Hadoop ecosystem, Parquet has gained widespread adoption due to its ability to optimize storage and query performance. + +[See this](https://motherduck.com/learn-more/why-choose-parquet-table-file-format/) diff --git a/Career/20251229180156-airflow_tasks.md b/Career/20251229180156-airflow_tasks.md new file mode 100644 index 0000000..20d16e6 --- /dev/null +++ b/Career/20251229180156-airflow_tasks.md @@ -0,0 +1,9 @@ +A task in the context of airflow is the basic unit of execution. They are arranged into [Dags](id:3b16e19a-62f0-4705-a451-a8790181941f) and then have upstream and downstream dependencies set between them in order to express the order in which they should be run. + +There are three kinds of tasks: + +1. Operators: they are predefined task templates that you can string together quickly. +2. Sensors: special subclass of operators, which are entirely about waiting for an external event to happen. +3. A Taskflow-decorated @task, which is a custom python function packaged as a task. + +[See this](https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/tasks.html) diff --git a/Career/20251229180249-airflow_dags.md b/Career/20251229180249-airflow_dags.md new file mode 100644 index 0000000..004a1ef --- /dev/null +++ b/Career/20251229180249-airflow_dags.md @@ -0,0 +1 @@ +A DAG (short for directed acyclic graphs) is a model that encapsulates everything needed to execute a workflow. [See this](https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/dags.html) diff --git a/Career/20260101011126-xor.md b/Career/20260101011126-xor.md new file mode 100644 index 0000000..1d5555d --- /dev/null +++ b/Career/20260101011126-xor.md @@ -0,0 +1,30 @@ +XOR stands for exclusive or. It is a logical operation that outputs true only when the inputs differ (one is true, the other is false). If both inputs are the same (both true or both false), the output is false. + +Here is the truth table for XOR: + +| | | | +| ----- | ----- | ------- | +| A | B | A XOR B | +| false | false | false | +| false | true | true | +| true | false | true | +| true | true | false | + +In programming, XOR is often represented by the caret symbol (^). For example, in many programming languages, you can use the XOR operator like this: + +``` python +a = True +b = False +result = a ^ b # result will be True +``` + +In simple terms it means "one or the other, but not both". + +Some similar concepts include: + + - OR (inclusive or): Outputs true if at least one input is true. + - AND: Outputs true only if both inputs are true. + - NOT: Outputs the opposite of the input (true becomes false, false becomes true). + - NAND: Outputs false only if both inputs are true (the opposite of AND). + - NOR: Outputs true only if both inputs are false (the opposite of OR). + - XNOR: Outputs true only when both inputs are the same (the opposite of XOR). diff --git a/Career/20260101012549-solid_principles.md b/Career/20260101012549-solid_principles.md new file mode 100644 index 0000000..7a07c07 --- /dev/null +++ b/Career/20260101012549-solid_principles.md @@ -0,0 +1,15 @@ +The SOLID principles are a set of five design principles that help software developers create maintainable, scalable, and flexible software systems. The principles were introduced by Robert C. Martin (Uncle Bob) and are widely used in object-oriented programming. + +The SOLID principles are: + +1. ****Single Responsibility Principle (SRP)****: A class should have only one reason to change, meaning it should have only one job or responsibility. This makes the class easier to understand and maintain. + +2. ****Open/Closed Principle (OCP)****: Software entities (classes, modules, functions, etc.) should be open for extension but closed for modification. This means that you should be able to add new functionality without changing existing code, which helps prevent bugs and maintain stability. + +3. ****Liskov Substitution Principle (LSP)****: Subtypes must be substitutable for their base types without altering the correctness of the program. This means that objects of a derived class should be able to replace objects of the base class without affecting the behavior of the program. + +4. ****Interface Segregation Principle (ISP)****: Clients should not be forced to depend on interfaces they do not use. This means that it's better to have many small, specific interfaces rather than a few large, general ones, which helps reduce dependencies and improve flexibility. + +5. ****Dependency Inversion Principle (DIP)****: High-level modules should not depend on low-level modules; both should depend on abstractions. Additionally, abstractions should not depend on details; details should depend on abstractions. This principle encourages the use of interfaces and abstract classes to decouple high-level and low-level components. + +[solidprinciplesexamples](id:F3875F0F-5C9E-4BB1-A79C-6000B9558115) diff --git a/Career/20260101012731-solid_principles_examples.md b/Career/20260101012731-solid_principles_examples.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/Career/20260101012731-solid_principles_examples.md @@ -0,0 +1 @@ + diff --git a/Career/20260115152626-apim.md b/Career/20260115152626-apim.md new file mode 100644 index 0000000..3895578 --- /dev/null +++ b/Career/20260115152626-apim.md @@ -0,0 +1,55 @@ +**Z notes on API management so I don’t forget** + +1. API stands for application programming interface. It allows systems to communicate with each other. +2. API management (or APIM) is the workstream that aims to provide a new technology platform for Microlise. It introduces a separate layer between customers and Microlise’s existing API products. + +The core pillars: + +![](./assets/career/Screenshot%202026-01-14%20154800.png) + +The workflow of publishing API’s: + +![](./assets/career/APIOps.png) + +This is summarised below: + +1. When someone wants to create an API, they do so by creating a repo and a C\# project. Ideally we should use the beautiful templates (). + +2. When the engineer wants to deploy this API, it goes through a gated and main pipeline. Phase 2 of the main build will build the image using quay registry so it’s available there (quay.mms.local). The release will consume the artifact produced by the main pipeline and will push these endpoints into the Openshift server. Essentially it will pull the image from the quay registry, containerise it + +Then publishing Api's into APIM (API management) using APIOps + +1. The engineer will need to setup swagger build (docs explains how to do this) +2. The engineer will create a PR for an API that has setup the APIOps pipeline (ie added the yaml pipeline and configured the variables - again, docs explain this in detail) +3. There are two pipelines associated with this: + - The gated build: [Link (Gated)](https://azdo.microlise.com/MicroliseCollection/Microlise/_taskgroup/c85996c9-d9ae-40ff-bd60-94b60db541a9) + - The main build: [Link (Main)](https://azdo.microlise.com/MicroliseCollection/Microlise/_taskgroup/7bd21d7f-499b-487d-a0f7-2206ffdff841) + - You will see that the term `spec` is used a lot. This is simply a json/yaml representation of your API interface (what you expose to the customer). + - One of the gated tasks is linting, it uses the scripts in this directory to test the OpenAPI specification linting, see: [Link](https://azdo.microlise.com/MicroliseCollection/Microlise/_git/ApiManagement.Pipeline.AgentScripts). +4. Another task adds API Governance Council as reviewers if there are OpenAPI Specification changes. All this can be viewed in the task group (see point a). +5. Once the main build passes, it triggers an ApiOps pipeline (defined in step 2), which creates a PR in the APIOps repository ([Link](https://azdo.microlise.com/MicroliseCollection/Microlise/_git/Microlise.APIOps)) +6. Once this PR is reviewed and approved by the Api GC, the Publish pipeline is triggered ([Link](https://azdo.microlise.com/MicroliseCollection/Microlise/_build?definitionId=1686&_a=summary)) which does a 2 way sync so that the APIM environments are in line with the OpenAPI specs. The publisher pipeline runs upto the CERT environment only, the PROD requires approval. + +There are two openshift clusters, one is gen and one is prod. Anything not prod goes into gen, and no one can access the prod cluster besides Openshift itself so test well. + +The link to the sharepoint site: + + - [Api management sharepoint](https://microliseuk.sharepoint.com/sites/SoftwareEngineering/SitePages/API-Management.aspx?source=https%3A%2F%2Fmicroliseuk.sharepoint.com%2Fsites%2FSoftwareEngineering%2FSitePages%2FForms%2FByAuthor.aspx) + +Link to the onboarding doc: + + - [Onboarding Doc](https://microliseuk.sharepoint.com/:w:/r/sites/SoftwareEngineering/_layouts/15/Doc.aspx?sourcedoc=%7BCF78DE96-7CC0-482A-9A66-647771CB8A04%7D&file=APIOpsOnboardingDocumentation.docx&action=default&mobileredirect=true) + +The link to the API management API gateway release: + + - [Link](https://azdo.microlise.com/MicroliseCollection/Microlise/_release?_a=releases&view=mine&definitionId=71) + +Reverse proxy: + + - [Pipeline](https://azdo.microlise.com/MicroliseCollection/Microlise/_build?definitionId=2439&_a=summary) + - [Sharepoint](https://microliseuk.sharepoint.com/sites/SoftwareEngineering/SitePages/Support-Guide---APIM-Reverse-Proxy.aspx) + +Read up on: + + - Quick Start Kubernetes (nigel Poulton) + - How is openshift different from k8? ([Link](https://microliseuk.sharepoint.com/sites/StorageCompute/ContainerPlatformUsers/SitePages/How-is-OpenShift-different-from-Kubernetes.aspx)) diff --git a/Career/20260121173318-api_architecture.md b/Career/20260121173318-api_architecture.md new file mode 100644 index 0000000..0fc964d --- /dev/null +++ b/Career/20260121173318-api_architecture.md @@ -0,0 +1,549 @@ +**Z notes on API architecture - companion to [APIM notes](id:56fabaf6-e8aa-45d0-a1c1-89f247f0a93f)** + +# 1\. What is API Architecture? + +API architecture is the set of rules, patterns, and structural decisions that govern how APIs are designed, exposed, consumed, and maintained across a system. It sits above individual API implementation - it's about **how APIs fit together** as a platform. + +In Microlise's context: the APIOps pipeline, the APIM gateway layer, OpenShift clusters, and the OpenAPI specs are all artefacts **of** an architectural decision. Understanding the architecture behind them makes the pipeline choices make sense. + +# 2\. Architectural Styles + +Different styles define how clients and servers communicate. These are not mutually exclusive - a platform can expose multiple styles simultaneously (e.g. REST externally, gRPC internally). + +## 2.1 REST (Representational State Transfer) + +The dominant style for public and partner APIs. Key constraints: + + - **Stateless**: Each request must contain all the context needed to fulfil it. No session state is stored server-side between calls. + + - **Resource-oriented**: APIs are modelled around nouns (resources), not verbs (actions). + + - Good: `GET /vehicles/{id}` + - Bad: `POST /getVehicle` + + - **Uniform interface**: Standard HTTP verbs carry semantic meaning: + + | Verb | Meaning | + | ------ | --------------------------- | + | GET | Read a resource | + | POST | Create a resource | + | PUT | Replace a resource entirely | + | PATCH | Partially update a resource | + | DELETE | Remove a resource | + + + - **Layered system**: Clients don't know if they're talking to the real backend or a gateway/proxy/cache. This is exactly what Microlise's APIM layer provides. + + - **Cacheable**: Responses should declare whether they can be cached, enabling CDN and client-side optimisation. + +OpenAPI Specification (OAS/Swagger) is the standard way to **describe** a REST API. The `spec` referred to throughout the APIOps pipeline is this document. + +## 2.2 GraphQL + +A query language for APIs developed by Meta. Instead of fixed endpoints, clients send a query describing exactly what data they need. + + - Single endpoint: `POST /graphql` + - Client drives the shape of the response - no over-fetching or under-fetching. + - Good for: complex, interconnected data models; front-end teams who iterate quickly. + - Trade-off: harder to cache, more complex server-side resolver logic, linting/governance is less mature than OAS. + +Not currently the Microlise APIM pattern but worth understanding as a contrast. + +## 2.3 gRPC (Google Remote Procedure Call) + +Uses Protocol Buffers (protobuf) as the interface definition language and HTTP/2 as transport. + + - Strongly typed contracts defined in `.proto` files. + - Extremely high performance - binary serialisation, multiplexed streams. + - Ideal for: internal service-to-service calls, microservices, high-throughput scenarios. + - Trade-off: not human-readable, harder to test with standard tooling (curl, Postman), less browser-friendly. + +Think of gRPC as what might live **behind** an API gateway - internal communication between microservices - while REST/OAS faces outward toward customers. + +## 2.4 AsyncAPI / Event-Driven APIs + +Not all APIs are request-response. Event-driven APIs use messaging patterns: + + - **Webhooks**: Server POSTs to a client-registered URL when an event occurs. + - **WebSockets**: Persistent bi-directional connection between client and server. + - **Server-Sent Events (SSE)**: One-way stream from server to client. + - **Message queues** (Kafka, RabbitMQ, Azure Service Bus): Decoupled async messaging. + +AsyncAPI is the OAS equivalent for event-driven interfaces - a specification format for documenting these contracts. + +# 3\. API Gateway Pattern + +This is the core of what APIM implements. An API gateway sits as an intermediary between consumers (customers, internal teams) and backend services. + +``` mermaid +flowchart TD + EC[External Consumer] + GW["API Gateway / APIM + ──────────────────── + Auth · Rate Limiting + Transforms · Routing + Logging · Caching"] + SA["Service A\n(OpenShift)"] + SB["Service B\n(OpenShift)"] + SC["Service C\n(OpenShift)"] + + EC --> GW + GW --> SA + GW --> SB + GW --> SC +``` + +## 3.1 What the gateway does + +| Concern | What it means | +| --------------------------- | ------------------------------------------------------------------------- | +| **Authentication** | Verifies who the caller is (OAuth2 tokens, API keys, mutual TLS) | +| **Authorisation** | Determines what the caller is allowed to do (scopes, claims) | +| **Rate limiting** | Caps requests per second/minute/hour per consumer or globally | +| **Throttling** | Gracefully slows or queues excess requests rather than rejecting them | +| **Request transformation** | Rewrites headers, payloads, or paths before forwarding to backends | +| \*Response transformation\* | Strips internal fields, reformats responses for the consumer contract | +| **Routing** | Directs traffic to the correct backend based on path, headers, or content | +| **Load balancing** | Distributes traffic across backend instances | +| **Caching** | Stores responses to reduce backend load for idempotent requests | +| **Observability** | Centralises access logs, metrics, and tracing across all APIs | + +In Azure APIM specifically, these concerns are implemented as **policies** - XML-based declarative rules that run at gateway level. + +## 3.2 Reverse Proxy vs API Gateway + +The Microlise notes reference a Reverse Proxy pipeline alongside APIM. These are related but distinct: + +| Aspect | Reverse Proxy | API Gateway | +| ------------------ | ------------------------------ | ---------------------------------------------- | +| Primary purpose | Routing and TLS termination | Full API lifecycle management | +| Protocol awareness | Layer 4/7 (TCP/HTTP) | Layer 7, API-aware (understands REST, OAS) | +| Policy engine | Minimal (Nginx/HAProxy config) | Rich (auth, transforms, quotas, subscriptions) | +| Developer portal | No | Yes - consumer-facing API catalogue | +| Examples | Nginx, HAProxy, Traefik | Azure APIM, Kong, AWS API Gateway | + +A common pattern (and likely what Microlise uses) is: Reverse proxy handles ingress and TLS termination → traffic forwarded to APIM for policy enforcement → APIM routes to OpenShift services. + +# 4\. API Design Principles + +## 4.1 Contract-First Design + +Define the OpenAPI spec **before** writing implementation code. The spec is the source of truth. + +Benefits: + + - Frontend/consumer teams can mock and build against the spec immediately. + - Linting pipelines (like the one in APIOps gated build) can enforce governance before any code ships. + - Breaking change detection is automated. + +The APIOps pipeline enforces this: the spec is committed to source control, linted, reviewed by the API Governance Council, and only then does the publish pipeline sync it to APIM environments. + +## 4.2 Versioning Strategies + +APIs evolve. Versioning prevents changes from breaking existing consumers. + +| Strategy | Example | Trade-offs | +| --------------------- | ------------------------------------- | --------------------------------------------------- | +| **URI versioning** | `/v1/vehicles`, `/v2/vehicles` | Explicit, cacheable, easy to route. Pollutes paths. | +| **Header versioning** | `Accept: application/vnd.api.v2+json` | Clean URIs. Harder to test, less cache-friendly. | +| **Query param** | `/vehicles?version=2` | Simple but considered poor practice for REST. | + +URI versioning is the most common and is what Azure APIM handles well via routing rules. + +## 4.3 Breaking vs Non-Breaking Changes + +Knowing what constitutes a breaking change is critical for API governance (i.e. why the API GC reviews spec PRs). + +| **Non-breaking (additive)** | **Breaking** | +| --------------------------------------- | ------------------------------------------------- | +| Adding a new optional field to response | Removing or renaming a field | +| Adding a new endpoint | Changing a field's type | +| Adding a new optional query parameter | Making an optional parameter required | +| New enum value (with caution) | Changing HTTP status codes for existing scenarios | +| | Changing authentication schemes | + +## 4.4 Resource Naming Conventions + + - Use **nouns**, not verbs: `/journeys` not `/getJourneys` + - Use **plural** for collections: `/vehicles` not `/vehicle` + - Use **kebab-case** for multi-word: `/driver-events` not `/driverEvents` + - Nest to show ownership, but limit depth: `/vehicles/{id}/journeys` is fine; `/vehicles/{id}/journeys/{jid}/events/{eid}/metadata` is not. + - Never expose internal implementation details in paths (`/{internalDatabaseId}` leaks schema). + +## 4.5 HTTP Status Codes + +Correct status codes are part of the API contract. Misuse breaks consumers who rely on them. + +| Code | Meaning | When to use | +| ---- | --------------------- | -------------------------------------------------- | +| 200 | OK | Successful GET, PUT, PATCH | +| 201 | Created | Successful POST that created a resource | +| 204 | No Content | Successful DELETE or action with no response body | +| 400 | Bad Request | Client sent malformed/invalid data | +| 401 | Unauthorized | Not authenticated (no or invalid token) | +| 403 | Forbidden | Authenticated but not authorised for this resource | +| 404 | Not Found | Resource does not exist | +| 409 | Conflict | State conflict (duplicate, version mismatch) | +| 422 | Unprocessable Entity | Semantically invalid (e.g. invalid date range) | +| 429 | Too Many Requests | Rate limit exceeded | +| 500 | Internal Server Error | Unhandled server-side failure | +| 503 | Service Unavailable | Downstream dependency down, circuit breaker open | + +# 5\. API Security Architecture + +## 5.1 Authentication Patterns + +| Pattern | How it works | Typical use | +| ------------------ | ------------------------------------------------------------------ | ------------------------------------------ | +| **API Keys** | Static key passed in header (`x-api-key`) or query param | Simple, internal/partner APIs | +| **OAuth 2.0** | Token-based; client obtains a bearer token from auth server | Public APIs, delegated access | +| **OpenID Connect** | OAuth 2.0 + identity layer (ID tokens, user info endpoint) | APIs that need to know **who** the user is | +| **Mutual TLS** | Both client and server present certificates | High-security service-to-service | +| **JWT** | Signed token carrying claims; verified without calling auth server | Stateless auth at gateway level | + +Azure APIM supports all of these via policies. A common pattern: APIM validates the JWT at the gateway before the request ever reaches an OpenShift pod. + +## 5.2 OAuth 2.0 Grant Types + +| Grant type | Use case | +| ----------------------------- | ------------------------------------------------------------ | +| **Client Credentials** | Machine-to-machine (no user involved). Most common for APIs. | +| **Authorization Code** | User-facing apps; user logs in and delegates access | +| **Authorization Code + PKCE** | Same as above but for SPAs/mobile (no client secret) | +| **Implicit** (deprecated) | Was used for SPAs - replaced by Auth Code + PKCE | + +## 5.3 Zero Trust at the API Layer + +Zero Trust means: **never trust, always verify** - even internal services must authenticate. + +Principles applied to APIs: + + - Every service-to-service call requires a valid token (no implicit trust on the internal network). + - Tokens have minimum required scopes (principle of least privilege). + - mTLS between internal services adds a second layer even if a token is compromised. + - All traffic - internal and external - goes through the gateway and is logged. + +# 6\. API Lifecycle Management + +This maps directly to the APIOps workflow in the APIM notes. + +``` mermaid +flowchart LR + Design[Design] + Develop[Develop] + Test[Test] + Publish[Publish] + Monitor[Monitor] + Retire[Retire] + + Design --> Develop --> Test --> Publish --> Monitor --> Retire + + Design -.-> D1["OAS Spec\nContract First"] + Develop -.-> D2["C# Project\nTemplates\nOpenShift"] + Test -.-> D3["Gated Build\nPipeline\nLinting + API GC"] + Publish -.-> D4["APIM Publish\nPipeline\nDev → Cert → Prod"] + Monitor -.-> D5["Analytics\nDashboards\nAPIM Portal"] + Retire -.-> D6["Deprecation\nNotices\nVersion Sunset"] +``` + +## 6.1 API Governance + +The API Governance Council (API GC) referenced in the notes is the enforcement body for architectural standards. Common governance concerns: + + - **Linting**: Automated rules against the OAS spec. The APIOps pipeline uses scripts from `ApiManagement.Pipeline.AgentScripts` to enforce this. + - **Review gates**: No spec change merges without GC approval - prevents inconsistent or insecure APIs reaching production. + - **Naming standards**: Enforced in the spec review (see §4.4). + - **Breaking change policy**: Defines how long old versions must be supported before retirement. + - **Security policy**: All APIs must use approved auth methods; no unauthenticated endpoints in production. + +## 6.2 APIOps (GitOps for APIs) + +APIOps applies GitOps principles to API management: the APIM configuration is stored as code in a Git repository and the pipeline is the only mechanism that changes APIM state. + +Key properties: + + - **Declarative**: The `Microlise.APIOps` repo describes the desired state of all APIs in APIM. + - **Versioned**: Every change is a PR - full audit trail. + - **Automated**: The publish pipeline does the two-way sync; no manual APIM portal edits. + - **Environment promotion**: Changes flow Dev -\> Cert -\> Prod, with a manual approval gate before Prod. + +This is analogous to how Terraform or Helm work for infrastructure - the repo **is** the truth. + +# 7\. API Observability + +An often-overlooked architectural concern. APIs you can't observe are APIs you can't operate. + +## 7.1 The Three Pillars + +| Pillar | What it captures | Tooling examples | +| ----------- | ----------------------------------------------------------- | ---------------------------------- | +| **Logs** | Discrete events: requests, responses, errors, auth failures | Azure Monitor, ELK, Splunk | +| **Metrics** | Aggregated numbers over time: latency, error rate, RPS | Prometheus, Azure Metrics, Grafana | +| **Traces** | End-to-end request journey across services | Jaeger, Zipkin, Azure App Insights | + +## 7.2 Key API Metrics to Track + + - **Latency**: p50, p95, p99 - not just average. Averages hide outliers. + - **Error rate**: 5xx rate (server errors) and 4xx rate (client errors) separately. + - **Throughput**: Requests per second - used to set rate limits and plan capacity. + - **Availability**: Uptime percentage. SLAs are usually defined here (99.9% = \~8.7h downtime/year). + - **Quota consumption**: How much of a consumer's rate limit are they using? + +Azure APIM exposes all of these natively and can emit them to Azure Monitor. + +## 7.3 Correlation IDs + +Every request should carry a unique `correlation-id` (or `x-request-id`) header. The gateway generates one if the client doesn't provide it and forwards it to all downstream services. This makes it possible to trace a single user request across multiple microservice logs. + +``` mermaid +flowchart LR + C[Client] + APIM["APIM\ngenerates correlation-id: abc-123"] + SA["Service A logs\nabc-123 · vehicle lookup"] + SB["Service B logs\nabc-123 · journey history"] + + C --> APIM + APIM --> SA + SA --> SB +``` + +# 8\. Microservices & API Design + +The OpenShift deployment model in Microlise's stack implies microservices. API architecture must account for how services communicate internally vs. externally. + +## 8.1 Internal vs External APIs + +| Aspect | Internal (East-West) | External (North-South) | +| --------------- | --------------------------------------- | ------------------------------------------ | +| Consumers | Other microservices | Customers, partners, third parties | +| Protocol | gRPC, internal REST, message queues | REST over HTTPS via APIM | +| Auth | mTLS, service accounts, internal tokens | OAuth2, API keys managed by APIM | +| Discoverability | Service mesh / internal DNS | Developer portal in APIM | +| Governance | Team-level conventions | API GC, formal versioning, SLA commitments | + +## 8.2 API Aggregation / BFF Pattern + +Backend for Frontend (BFF): a dedicated API layer tailored to a specific consumer (e.g. a mobile app, a portal). Instead of the consumer calling 5 microservices, a BFF aggregates them into a single call. + +``` mermaid +flowchart LR + MA[Mobile App] + BFF[BFF: Mobile API] + VS[Vehicle Service] + JS[Journey Service] + DS[Driver Service] + + MA --> BFF + BFF --> VS + BFF --> JS + BFF --> DS +``` + +APIM policies can implement lightweight aggregation, but for complex cases a dedicated BFF service is cleaner. + +## 8.3 Service Mesh (Complementary to API Gateway) + +A service mesh (e.g. Istio, Linkerd) manages **internal** service-to-service communication within OpenShift/Kubernetes: + + - mTLS between pods automatically. + - Traffic policies (retries, circuit breaking) at the network level. + - Observability (traces, metrics) without code changes. + +The API Gateway handles North-South (external) traffic; the service mesh handles East-West (internal). They are complementary, not competing. + +# 9\. API Design Patterns + +## 9.1 Pagination + +Never return unbounded collections. Standard patterns: + + - **Offset/limit**: `GET /journeys?offset=0&limit=50`. Simple but inefficient at high offsets. + - **Cursor-based**: `GET /journeys?cursor=eyJpZCI6MTAwfQ==`. Efficient for large datasets; the cursor encodes the last seen position. + - **Page-based**: `GET /journeys?page=3&pageSize=50`. User-friendly but shares offset's inefficiency. + +Response should include metadata: + +``` json +{ + "data": [...], + "pagination": { + "total": 1420, + "limit": 50, + "nextCursor": "eyJpZCI6MTUwfQ==" + } +} +``` + +## 9.2 Filtering, Sorting, and Field Selection + + - Filtering: `GET /vehicles?status=active&driverType=HGV` + - Sorting: `GET /journeys?sort=-startedAt` (prefix `-` for descending) + - Field selection (sparse fieldsets): `GET /vehicles?fields=id,registration,status` - reduces payload size. + +## 9.3 Idempotency + +A request is idempotent if making it multiple times produces the same result as making it once. Crucial for retry logic. + +| Method | Idempotent? | Safe (no side effects)? | +| ------ | ----------- | ----------------------- | +| GET | Yes | Yes | +| PUT | Yes | No | +| DELETE | Yes | No | +| POST | No | No | +| PATCH | No\* | No | + +\*PATCH can be designed to be idempotent but isn't by definition. + +For non-idempotent operations (POST), use an `Idempotency-Key` header. The server stores the result keyed to that value; duplicate requests return the cached result rather than processing again. + +## 9.4 HATEOAS + +Hypermedia as the Engine of Application State - responses include links to related actions: + +``` json +{ + "id": "v-123", + "registration": "AB12 CDE", + "_links": { + "self": { "href": "/vehicles/v-123" }, + "journeys": { "href": "/vehicles/v-123/journeys" }, + "driver": { "href": "/drivers/d-456" } + } +} +``` + +Rarely implemented fully in practice but worth understanding as the most complete expression of REST. + +# 10\. OpenAPI Specification Deep Dive + +Since OAS is central to the APIOps pipeline, understanding its structure is practical knowledge. + +``` yaml +openapi: "3.1.0" +info: + title: Vehicle Service API + version: "2.0.0" + description: Manages vehicle records for the Microlise platform. + +servers: + - url: https://api.microlise.com/v2 + description: Production + +paths: + /vehicles/{vehicleId}: + get: + summary: Get a vehicle by ID + operationId: getVehicleById # Unique identifier used in code gen + tags: [Vehicles] + parameters: + - name: vehicleId + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: Vehicle found + content: + application/json: + schema: + $ref: "#/components/schemas/Vehicle" + "404": + $ref: "#/components/responses/NotFound" + +components: + schemas: + Vehicle: + type: object + required: [id, registration] + properties: + id: + type: string + format: uuid + registration: + type: string + example: "AB12 CDE" + status: + type: string + enum: [active, inactive, maintenance] + + responses: + NotFound: + description: Resource not found + content: + application/json: + schema: + $ref: "#/components/schemas/ProblemDetails" + + securitySchemes: + oauth2: + type: oauth2 + flows: + clientCredentials: + tokenUrl: https://auth.microlise.com/oauth2/token + scopes: + vehicles:read: Read vehicle data + vehicles:write: Create and update vehicles + +security: + - oauth2: [vehicles:read] +``` + +Key OAS concepts: + + - `operationId`: Used by code generators and APIM to reference operations in policies. + - `$ref`: DRY principle - define schemas and responses once, reuse everywhere. + - `components`: The library section - schemas, parameters, responses, security schemes. + - `tags`: Grouping for the developer portal - consumers see organised API docs. + - `security`: Applied globally here; can be overridden per-operation. + +# 11\. Connecting the Dots: Microlise Architecture Map + +Mapping the APIM notes to the architectural concepts above: + +| APIM Note Item | Architectural Concept | +| ---------------------------------------- | -------------------------------------------------- | +| Separate layer between customers & APIs | API Gateway Pattern (§3) | +| OpenAPI spec / swagger build | Contract-First Design (§4.1), OAS (§10) | +| APIOps pipeline, spec in git | APIOps / GitOps for APIs (§6.2) | +| API Governance Council review | API Governance (§6.1), Breaking Changes (§4.3) | +| Linting scripts (`AgentScripts`) | Automated governance enforcement | +| Gated + main pipeline | CI/CD gates for quality and security | +| Two-way sync, publish pipeline | Declarative state management (APIOps) | +| Dev -\> Cert -\> Prod with approval gate | Environment promotion pattern (§6.2) | +| Gen cluster vs Prod cluster | Environment isolation, blast radius reduction | +| Reverse proxy pipeline | Reverse Proxy vs API Gateway (§3.2) | +| Quay registry, OpenShift containers | Microservices deployment, East-West traffic (§8.1) | +| Swagger/OAS spec as PR artefact | Spec-as-code, version-controlled contracts | + +# 12\. Further Reading & Reference + +## Recommended (from APIM notes) + + - Quick Start Kubernetes - Nigel Poulton ([OpenShift vs K8s](https://microliseuk.sharepoint.com/sites/StorageCompute/ContainerPlatformUsers/SitePages/How-is-OpenShift-different-from-Kubernetes.aspx)) + +## Additional Architecture Resources + + - [OpenAPI Specification 3.1.0 (official)](https://spec.openapis.org/oas/v3.1.0) + - [AsyncAPI Documentation](https://www.asyncapi.com/docs) + - [gRPC Official Docs](https://grpc.io/docs/) + - [Azure API Management Docs](https://learn.microsoft.com/en-us/azure/api-management/) + - [Microservices.io - API Gateway Pattern](https://microservices.io/patterns/apigateway.html) + - [OAuth 2.0 (oauth.net)](https://oauth.net/2/) + - [Swagger / OAS Reference](https://swagger.io/specification/) + +## Key Terms Glossary + +| Term | Definition | +| --------------- | ---------------------------------------------------------------------------- | +| OAS / Swagger | OpenAPI Specification - a standard format for describing REST APIs | +| APIOps | Applying GitOps principles to API management (spec-as-code, pipeline-driven) | +| APIM | API Management - the platform/layer that governs API lifecycle | +| Gateway | Intermediary that enforces policy (auth, rate limiting, routing) for APIs | +| Spec | Short for specification - the OAS JSON/YAML document describing an API | +| Idempotency | Property where repeating a request has the same effect as making it once | +| mTLS | Mutual TLS - both parties in a connection authenticate with certificates | +| BFF | Backend for Frontend - an API tailored to a specific consumer's needs | +| Service Mesh | Infrastructure layer managing internal service-to-service communication | +| Breaking Change | An API change that requires existing consumers to update their integration | diff --git a/Career/20260305102210-cicd_example.md b/Career/20260305102210-cicd_example.md new file mode 100644 index 0000000..1e074db --- /dev/null +++ b/Career/20260305102210-cicd_example.md @@ -0,0 +1,546 @@ +During the first wave in 2026, we planned to get the netcore version of site visits out to customers. There were a bunch of tasks relating to this, so I asked AI to conceptually explain them. + +# Prompt: + +I want to understand builds and pipelines in more depth. we have a set of tasks to get a new upgraded project to customers. this is task 1: + +Add the .net core version to the TMC release pipeline. +Follow this guide so that the build artifacts are included in the TMC release + + + +(This will add the artifact to the release, you will then need to test this locally, see the other task) + +After that, we need to make the below changes as we did for the DriverWebAPI: + + + + + +this is task 2: +Add application file and make deployable locally +See this PR from when we previously did on Drivers: + +Note: That some of the properties in the files are different to what you have for a local deploy, if in doubt refer to main to ensure that we have no consistencies. + +Also, we don't need to alter the connection string as this has already been done. + +`==` + +This requires the previous tasks to be completed (main build and artifact added to release) + +Follow this commit as an example + + + +Create a new application file for the dotnet core version of the service, swap it out in the manifests for the local deploy along with any relevant configuration + +There may be some differences due to this being a web api rather than a background service, and the fact that we use the appsettings.json rather than the app.config - in which case look to Arrivals and departures and vehiclev2 if that is still around + +Set up local TMC and deploy - make sure it works as expected + +This is now possible on the new domain joined machines with a VM - did it myself and it works well + + + +this is 3: +Make deployable in QA (Toblerone) +Check out this commit for the AVL where the new service was added to the templates and the deploy file, use what was learned from the local deploy to ensure this works in the same way + +Check if you can just apply this to the toblerone box, the deployall.yml file governs what is actually deployed, this might not be possible but worth a check + +Test that Toblerone deploys okay + + + +See this PR for how we did it last time for the Driver Web API: + +This needs to be changed for the current release, potentially the previous release if it errors when that happens as well as the ad hoc. See the last three files in the above PR. + +task 4: +Add artifact to the ad-hoc pipelines for QA / Cert +Need to complete the previous Toblerone QA task first\! + +We need to add a similar artifact for SiteVisits to the ad hoc QA / Cert release pipeline like so: + +Image + +And then give it a run and make sure it works. + +See the release pipeline here: TMC.SiteVisitsWebApi - Pipelines + +task 5: +Ensure that you test each endpoint with different data, ensuring the experience is exactly the same for each endpoint (auth, request / response) + +As we are planning to hotswap the instances over, they must work exactly the same + +Make sure its on the same URL + +task 6: +Make deployable in the rest of QA +We have this running in Toblerone QA, once we are happy with the API in Toblerone we need to roll it out to the rest of QA. + +Aaron also suggested considering doing the internal CERT environments at this stage, then we have a more live like environment tested early on too. + +See this PR for how we did this for the Driver Web API: + +task 7: +Ensure that the new Serilog logs are captured into elastic +This file needs updating with the paths for the serilogs in the sitevisitswebapi + + + +However it does not look like servicename and customername parsing is working for the existing logs + +Effectively, add a line like this but for our SiteVisits API: + +Image + +task 8: +Make deployable in UAT +Update UAT to use the new dotnet core version of the API, use what was learned from the local deploy to ensure this works in the same way + +Use this commit for reference + + + +See this PR for what we did last time for UAT (incorporates other environments): + +task 9: +Make deployable in live +Update live to use the new dotnet core version of the API, use what was learned from the local deploy to ensure this works in the same way + +Use this commit for reference + + + +See this PR for what we did last time for live (incorporates other environments): + +# Response + +You’re essentially looking at a ****full CI/CD path**** for introducing a ****new version of a service (.NET Core API)**** into a large enterprise deployment system. The tasks look complicated because they touch ****multiple layers of the delivery pipeline****, not just code. + +I'll explain this in a ****mental model first****, then walk through ****what each task is really doing in the pipeline****. + +— + +## 1\. The Big Picture: What a Build & Release Pipeline Is + +In enterprise systems like Azure DevOps, deployment is usually split into two major stages. + +1. Build Pipeline (CI) + +Purpose: ****Turn source code into deployable artifacts**** + +Steps typically include: + + Code → Restore packages → Compile → Run tests → Package → Publish artifact + +Output = ****artifact**** + +Example artifact: + + SiteVisitsWebApi.zip + +This artifact contains: + + dlls + configs + dependencies + startup files + +Think of an ****artifact**** as a ****versioned package of your application****. + +1. Release Pipeline (CD) + +Purpose: ****Take the artifact and deploy it to environments**** + +Example environments: + + Local + QA + CERT + UAT + LIVE + +Each environment might have: + + - different configs + - different servers + - different manifests + +**Your System Specifically** + +You are working with something like: + +``` + + +Git Repo + ↓ +Build Pipeline + ↓ +Artifacts + ↓ +TMC Release Pipeline + ↓ +Deployment Manifests + ↓ +Servers + +``` + +## 2\. Why This Task Exists + +You are ****introducing a new .NET Core version of an API****. + +Previously there was likely: + + SiteVisitsWebApi (.NET Framework) + +Now: + + SiteVisitsWebApi (.NET Core) + +So you must: + +1. Build it +2. Package it +3. Add it to release pipeline +4. Update deployment configs +5. Deploy to environments gradually + +## 3\. What Each Task Actually Means + +I'll translate the tasks into ****plain engineering intent****. + +### Task 1: Add .NET Core Version to Build + Release + +What is happening? + +You add the ****new project to the build pipeline**** so it produces an artifact. + +Before: + +`Build → DriverWebAPI artifact` + +After: + +``` + +Build → DriverWebAPI artifact + → SiteVisitsWebApi artifact + +``` + +Why the PRs exist + +These repositories control deployment: + + TMC_Release + Deployment + TMCBranchTool + +They likely manage: + +| | | +| --------------------- | ------------------------- | +| Repo | Purpose | +| ————- | ————————- | +| TMCRelease | release packaging | +| Deployment | deployment scripts | +| TMCBranchTool | release branch automation | + +What ReleasePackagesConfig.csv probably does + +Something like: + +``` + +ServiceName,ArtifactName,DeploymentType +DriverWebApi,DriverWebApi.zip,WebApi +SiteVisitsWebApi,SiteVisitsWebApi.zip,WebApi + +``` + +So the pipeline knows: + + Include this artifact in the release bundle + +Result of Task 1 + +Your pipeline will now produce: + +Artifacts: + +`DriverWebApi` + +`SiteVisitsWebApi` + +And release pipeline can access them. + +### Task 2: Make Deployable Locally + +Now you test deployment ****without affecting real environments****. + +This uses ****deployment manifests****. + +What a manifest is + +A manifest is basically a ****deployment recipe****. + +Example: + +``` yaml + +apps: + - name: sitevisitswebapi + artifact: SiteVisitsWebApi + port: 5000 + config: appsettings.json + +``` + +It tells the deployment system: + +deploy this artifact +with these configs +to this environment + +What you do here + +Create a new ****application definition****. + +Example: + +`_apps.yml` + +Add: + +`sitevisitswebapi` + +Then configure: + + artifact source + ports + configs + environment variables + +Why they mention differences + +Your service is: + +`Web API` + +instead of + +`Background Windows Service` + +So deployment might differ: + +| | | +| ------------------ | ----------------------- | +| Background service | Web API | +| —————— | ———————– | +| runs as service | hosted in IIS / kestrel | +| app.config | appsettings.json | + +Result of Task 2 + +You can now run: + +local TMC deployment + +and your API launches locally. + +### Task 3: Deploy to QA (Toblerone) + +Now you move from: + +Local → QA + +"Toblerone" appears to be a ****specific QA environment/server****. + +You update manifests like: + +deployall.yml + +Which probably contains: + +``` yaml + + +apps: + - driverwebapi + - sitevisitswebapi +``` + +So the deployment engine knows: + +deploy this service in QA + +Then test deployment. + +### Task 4: Add to Ad-hoc QA / Cert Pipelines + +These are ****manual release pipelines****. + +Example use: + +`Dev: deploy specific service to QA quickly` + +Instead of full release. + +You add the artifact there so people can manually deploy it. + +### Task 5: Endpoint Testing + +Critical step. + +You must confirm the new API behaves ****exactly like the old one****. + +Because they plan a ****hotswap****. + +Meaning: + +Old API → replace with new API +same URL +same behaviour + +If anything changes, clients break + +So test: + +Auth +Request formats +Response formats +Error codes + +### Task 6: Deploy to Remaining QA + +Once Toblerone works: + +Toblerone QA → All QA environments + +Example: + +QA1 +QA2 +QA3 + +Maybe also: + +Internal CERT + +### Task 7: Logging (Serilog → Elastic) + +The new API logs differently. + +Probably something like: + +`logs/sitevisitswebapi/*.log` + +Filebeat must be told to collect them. + +Example config: + + - type: log + paths: + - /logs/sitevisitswebapi/*.log + +Then logs go: + +`Server → Filebeat → Elasticsearch → Kibana` + +### Task 8: Deploy to UAT + +UAT = staging environment. + +Production-like testing + +Here you update the deployment manifests to use the ****new API****. + +### Task 9: Deploy to Live + +Final production rollout. + +Same process: + +manifest updates +deploy pipeline +verification + +## 4\. The Entire Flow (Visual) + +``` + + +Code (SiteVisitsWebApi) + │ + ▼ +Build Pipeline + │ + ▼ +Artifact Produced +(SiteVisitsWebApi.zip) + │ + ▼ +TMC Release Pipeline + │ + ▼ +Deployment Manifests + │ + ▼ +Local Deploy + │ + ▼ +QA (Toblerone) + │ + ▼ +All QA + │ + ▼ +UAT + │ + ▼ +LIVE +``` + +## 5\. Why Enterprise Pipelines Feel Complicated + +Because deployment is separated into ****many independent systems****: + +| | | +| ------------------ | ----------------------- | +| Layer | Responsibility | +| —————— | ———————– | +| Build pipeline | compile code | +| Artifact storage | store packages | +| Release pipeline | orchestrate deployments | +| Manifests | describe environments | +| Deployment scripts | actually install | +| Logging | monitor | + +Each lives in ****different repos****. + +## 6\. The Most Important Concept to Understand + +Everything revolves around ****artifacts****. + + Source Code + ↓ + Build + ↓ + Artifact + ↓ + Deploy artifact everywhere + +You ****never deploy source code directly****. + +## 7\. What You're Really Doing in This Ticket + +You are teaching the deployment system: + +"Hey, there is a new service now" + +Then gradually making every environment aware of it. diff --git a/Career/20260305111610-cicd_summary.md b/Career/20260305111610-cicd_summary.md new file mode 100644 index 0000000..e071808 --- /dev/null +++ b/Career/20260305111610-cicd_summary.md @@ -0,0 +1,62 @@ +# Summary + +CI/CD stands for Continuous Integration and Continuous Delivery/Deployment. It’s a development practice that automates building, testing, and delivering software so code changes can be released quickly and reliably. + +## Continuous Integration (CI) + +CI means developers frequently merge their code into a shared repository. Every time code is pushed: + +1. The code is built automatically. +2. Automated tests run. +3. If something breaks, the team is alerted. + +Goal: catch bugs early and ensure new code works with the rest of the project. + +## Continuous Delivery / Continuous Deployment (CD) + +CD takes the tested code from CI and prepares it for release. + + - Continuous Delivery: Code is automatically prepared for production, but a human approves the final deployment. + + - Continuous Deployment: Code is automatically deployed to production with no manual approval. + +Goal: make releases faster and safer. + +## Pipelines + +A pipeline is the automated workflow that runs these steps in sequence. Think of it as a script that defines what happens after a code change. + +Typical pipeline stages: + +1. Source – code pushed to repository +2. Build – compile the application +3. Test – run automated tests +4. Package – create deployable artifact (e.g., Docker image) +5. Deploy – release to staging or production + +Example simplified pipeline: + +`Code Push → Build → Test → Package → Deploy` + +Why CI/CD is useful + + - Faster development cycles + + - Fewer integration bugs + + - Automated testing and deployment + + - More reliable releases + +Common CI/CD tools + + - GitHub Actions + + - Jenkins + + - GitLab CI/CD + + - CircleCI + +In short: +CI/CD uses pipelines to automatically build, test, and deploy software whenever code changes are made. diff --git a/Career/20260305130457-database_perms_roles_accounts.md b/Career/20260305130457-database_perms_roles_accounts.md new file mode 100644 index 0000000..c9de58e --- /dev/null +++ b/Career/20260305130457-database_perms_roles_accounts.md @@ -0,0 +1,238 @@ +# Introduction + +Database security is a crucial part of database administration. It ensures that only authorised users can access, modify, or manage data. Three core concepts used to control access are: + +1. Accounts (logins/users) +2. Roles +3. Permissions + +These concepts work together to create structured and secure access control inside a database system. + +# 1\. Database Accounts + +A **database account** represents an identity that can connect to the database system. In Microsoft SQL Server this is typically split into two layers: + + - **Login** → Authentication at the server level + - **User** → Authorisation inside a specific database + +## Login (Server Level) + +A login allows someone or something to authenticate with the SQL Server instance. + +Example: Creating a login + +``` sql +CREATE LOGIN student_user +WITH PASSWORD = 'StrongPassword123!'; +``` + +You can also create a login linked to Windows authentication. + +``` sql +CREATE LOGIN [DOMAIN\Zaine] FROM WINDOWS; +``` + +## Database User + +A login must be mapped to a user inside a database before it can access that database. + +Example: + +``` sql +USE SchoolDB; + +CREATE USER student_user +FOR LOGIN student_user; +``` + +Now the login can access the **SchoolDB** database as the user **studentuser**. + +# 2\. Permissions + +Permissions define **what actions a user can perform**. These actions include reading data, inserting rows, modifying tables, or executing procedures. + +Common SQL Server permissions include: + + - SELECT → Read data + - INSERT → Add new data + - UPDATE → Modify data + - DELETE → Remove data + - EXECUTE → Run stored procedures + - ALTER → Modify database objects + - CONTROL → Full control over an object + +## Granting Permissions + +Permissions are given using the **GRANT** statement. + +Example: Allow a user to read data from a table. + +``` sql +GRANT SELECT +ON Students +TO student_user; +``` + +## Grant Multiple Permissions + +``` sql +GRANT SELECT, INSERT +ON Students +TO student_user; +``` + +This allows the user to read and add new rows. + +## Revoking Permissions + +If a permission should be removed: + +``` sql +REVOKE INSERT +ON Students +FROM student_user; +``` + +## Denying Permissions + +A **DENY** explicitly blocks an action, even if another role grants it. + +``` sql +DENY DELETE +ON Students +TO student_user; +``` + +# 3\. Roles + +Roles are collections of permissions that can be assigned to multiple users. They simplify permission management by allowing administrators to assign permissions once and reuse them. + +Instead of granting permissions to many individual users, you grant them to a role. + +Example scenario: + + - Many students should be able to view course data. + - Instead of assigning permissions to each student individually, create a role. + +## Creating a Role + +``` sql +CREATE ROLE student_role; +``` + +## Assign Permissions to the Role + +``` sql +GRANT SELECT +ON Courses +TO student_role; +``` + +## Add Users to the Role + +``` sql +ALTER ROLE student_role +ADD MEMBER student_user; +``` + +Now **studentuser** inherits all permissions from **studentrole**. + +# 4\. Built-in Database Roles + +SQL Server includes several predefined roles that already have common permission sets. + +Examples: + +| Role Name | Purpose | +| ----------------------- | --------------------------------- | +| dbowner | Full control over the database | +| dbdatareader | Read all tables | +| dbdatawriter | Insert/update/delete all tables | +| dbddladmin | Create or modify database objects | + +Example: Add a user to the read-only role. + +``` sql +ALTER ROLE db_datareader +ADD MEMBER student_user; +``` + +This allows the user to read all tables without giving modification rights. + +# 5\. Example: Simple University Database Security + +Assume a database called **UniversityDB** with two tables: + + - Students + - Courses + +Goal: + + - Students → Read course information + - Teachers → Modify course data + - Admin → Full control + +## Step 1: Create Roles + +``` sql +CREATE ROLE student_role; +CREATE ROLE teacher_role; +CREATE ROLE admin_role; +``` + +## Step 2: Assign Permissions + +Student role (read-only): + +``` sql +GRANT SELECT +ON Courses +TO student_role; +``` + +Teacher role: + +``` sql +GRANT SELECT, INSERT, UPDATE +ON Courses +TO teacher_role; +``` + +Admin role: + +``` sql +GRANT CONTROL +ON DATABASE::UniversityDB +TO admin_role; +``` + +## Step 3: Add Users + +``` sql +ALTER ROLE student_role ADD MEMBER student_user; +ALTER ROLE teacher_role ADD MEMBER teacher_user; +ALTER ROLE admin_role ADD MEMBER admin_user; +``` + +Now permissions are organised through roles instead of assigning them individually. + +# 6\. Why Roles Are Important + +Roles provide several benefits: + + - **Simpler management** → Change permissions in one place + - **Scalability** → Works well with many users + - **Security consistency** → Reduces risk of incorrect permissions + - **Easier auditing** → Clear structure of access control + +Without roles, administrators would need to manually manage permissions for every individual user. + +# Summary + +Database access control relies on three key components: + + - **Accounts** identify who is accessing the system (logins and users). + - **Permissions** define what actions can be performed. + - **Roles** group permissions together for easier management. + +In Microsoft SQL Server, administrators typically create logins, map them to database users, assign them to roles, and grant permissions to those roles. This layered approach ensures a secure and manageable database system. diff --git a/Career/20260305132058-restful_api.md b/Career/20260305132058-restful_api.md new file mode 100644 index 0000000..9f800e2 --- /dev/null +++ b/Career/20260305132058-restful_api.md @@ -0,0 +1,296 @@ +# What is a RESTful API? + +A **RESTful API** is a web service that follows the principles of **REST (Representational State Transfer)**. REST is an architectural style used for designing networked applications. + +In a RESTful system: + + - Everything is treated as a **resource** + - Resources are identified using **URLs** + - Standard **HTTP methods** are used to interact with resources + - Communication is usually done using **JSON** + +Example resource: + + /users + /users/1 + /users/1/orders + +These represent data stored on the server. + +# Core HTTP Methods + +REST APIs rely heavily on HTTP verbs. + +| Method | Purpose | Example Endpoint | +| ------ | --------------------------- | ---------------- | +| GET | Retrieve data | GET /users | +| POST | Create a new resource | POST /users | +| PUT | Update an existing resource | PUT /users/1 | +| DELETE | Remove a resource | DELETE /users/1 | + +# Example Resource: User + +Assume we have a simple **User** resource: + +``` json +{ + "id": 1, + "name": "Alice", + "email": "alice@email.com" +} +``` + +The API allows clients to create, read, update, and delete users. + +# Creating a REST API in C\# (ASP.NET Core) + +In C\#, REST APIs are commonly built using **ASP.NET Core Web API**. + +Example project creation: + +``` bash +dotnet new webapi -n UserApi +cd UserApi +dotnet run +``` + +This creates a ready-to-run REST API project. + +# Defining a Model + +First, define the resource model. + +File: Models/User.cs + +``` csharp +namespace UserApi.Models +{ + public class User + { + public int Id { get; set; } + + public string Name { get; set; } + + public string Email { get; set; } + } +} +``` + +This represents the data stored and returned by the API. + +# Creating a Controller + +Controllers handle HTTP requests. + +File: Controllers/UserController.cs + +``` csharp +using Microsoft.AspNetCore.Mvc; +using UserApi.Models; + +namespace UserApi.Controllers +{ + [ApiController] + [Route("api/[controller]")] + public class UserController : ControllerBase + { + private static List users = new List() + { + new User { Id = 1, Name = "Alice", Email = "alice@email.com" }, + new User { Id = 2, Name = "Bob", Email = "bob@email.com" } + }; + + [HttpGet] + public ActionResult> GetUsers() + { + return Ok(users); + } + } +} +``` + +Endpoint created: + + GET /api/user + +Response: + +``` json +[ + { "id": 1, "name": "Alice", "email": "alice@email.com" }, + { "id": 2, "name": "Bob", "email": "bob@email.com" } +] +``` + +# Getting a Single Resource + +Add an endpoint to retrieve a specific user. + +``` csharp +[HttpGet("{id}")] +public ActionResult GetUser(int id) +{ + var user = users.FirstOrDefault(u => u.Id == id); + + if (user == null) + { + return NotFound(); + } + + return Ok(user); +} +``` + +Endpoint: + + GET /api/user/1 + +# Creating a Resource (POST) + +Clients send JSON data to create a new user. + +``` csharp +[HttpPost] +public ActionResult CreateUser(User newUser) +{ + newUser.Id = users.Max(u => u.Id) + 1; + users.Add(newUser); + + return CreatedAtAction(nameof(GetUser), new { id = newUser.Id }, newUser); +} +``` + +Example request: + +``` json +POST /api/user + +{ + "name": "Charlie", + "email": "charlie@email.com" +} +``` + +# Updating a Resource (PUT) + +``` csharp +[HttpPut("{id}")] +public IActionResult UpdateUser(int id, User updatedUser) +{ + var user = users.FirstOrDefault(u => u.Id == id); + + if (user == null) + { + return NotFound(); + } + + user.Name = updatedUser.Name; + user.Email = updatedUser.Email; + + return NoContent(); +} +``` + +Endpoint: + + PUT /api/user/1 + +# Deleting a Resource + +``` csharp +[HttpDelete("{id}")] +public IActionResult DeleteUser(int id) +{ + var user = users.FirstOrDefault(u => u.Id == id); + + if (user == null) + { + return NotFound(); + } + + users.Remove(user); + + return NoContent(); +} +``` + +Endpoint: + + DELETE /api/user/1 + +# REST Principles + +A good REST API should follow these key ideas: + +## 1\. Statelessness + +Each request contains all information needed. + +The server does **not store client session state**. + +## 2\. Resource-Based URLs + +Endpoints should represent **nouns**, not verbs. + +Good: + + GET /users + POST /users + GET /users/1 + +Bad: + + GET /getUsers + POST /createUser + +## 3\. Standard HTTP Status Codes + +| Code | Meaning | +| ---- | ------------ | +| 200 | OK | +| 201 | Created | +| 204 | No Content | +| 400 | Bad Request | +| 404 | Not Found | +| 500 | Server Error | + +# Example Full API Structure + + UserApi/ + ├── Controllers/ + │ └── UserController.cs + ├── Models/ + │ └── User.cs + ├── Program.cs + └── appsettings.json + +# Testing the API + +You can test APIs using tools like: + + - curl + - Postman + - Swagger UI (included with ASP.NET) + +Example curl request: + +``` bash +curl http://localhost:5000/api/user +``` + +# Summary + +A RESTful API: + + - Exposes **resources via URLs** + - Uses **HTTP methods (GET, POST, PUT, DELETE)** + - Communicates typically using **JSON** + - Is **stateless** + - Returns **standard HTTP status codes** + +In C\#, **ASP.NET Core Web API** makes building REST APIs straightforward using: + + - Models + - Controllers + - Routing + - Built-in JSON serialization diff --git a/Career/20260327114708-asp_net_core_6_apis_notes.md b/Career/20260327114708-asp_net_core_6_apis_notes.md new file mode 100644 index 0000000..1e309d3 --- /dev/null +++ b/Career/20260327114708-asp_net_core_6_apis_notes.md @@ -0,0 +1 @@ +****Notes from [Pluralsight](https://app.pluralsight.com/ilx/video-courses/asp-dot-net-core-6-web-api-fundamentals/resources)**** diff --git a/Career/20260328215014-software_dev_methodologies.md b/Career/20260328215014-software_dev_methodologies.md new file mode 100644 index 0000000..ebd4ae6 --- /dev/null +++ b/Career/20260328215014-software_dev_methodologies.md @@ -0,0 +1,337 @@ +# Software Development Methodologies + +A software development methodology defines the *process, structure, and principles* +used to plan, manage, and execute software projects. + +## Why It Matters + + - Ensures predictability and consistency across teams + - Reduces risk of project failure + - Aligns stakeholders and developers on expectations + - Provides tools for handling change and uncertainty + +----- + +# 1\. Waterfall + +## Overview + +A **linear, sequential** model where each phase must be completed before the next begins. + +## Phases + +1. Requirements +2. System Design +3. Implementation +4. Testing +5. Deployment +6. Maintenance + +## Example + +``` example +A government agency contracts a firm to build a tax portal. +Requirements are frozen at sign-off; code is written six months +later; testing follows; the portal launches two years in. +``` + +## Pros & Cons + +| Pros | Cons | +| --------------------------- | ----------------------------------- | +| Clear structure and phases | Very rigid; change is costly | +| Easy to manage milestones | Late discovery of defects | +| Good for fixed requirements | Customer sees nothing until the end | + +## Best For + + - Projects with well-defined, stable requirements + - Regulatory/compliance-heavy environments (aerospace, medical devices) + +----- + +# 2\. Agile + +## Overview + +An **iterative, incremental** philosophy that values: + +> Individuals and interactions over processes and tools. +> Working software over comprehensive documentation. +> Customer collaboration over contract negotiation. +> Responding to change over following a plan. +> — The Agile Manifesto (2001) + +## Core Concepts + + - **Iterations (sprints)**: Short delivery cycles (1–4 weeks) + - **Incremental delivery**: Working software shipped frequently + - **Feedback loops**: Customers review and redirect regularly + - **Cross-functional teams**: Dev, QA, design collaborate continuously + +## Example + +``` example +A startup builds a mobile app in 2-week sprints. +Sprint 1 → user login +Sprint 2 → profile page +Sprint 3 → search feature +After each sprint, real users test and give feedback, +shaping what gets built next. +``` + +## Agile Values vs Waterfall + +| Dimension | Waterfall | Agile | +| ------------- | -------------- | -------------------- | +| Planning | Upfront, fixed | Continuous, adaptive | +| Delivery | End of project | Every sprint | +| Change | Discouraged | Embraced | +| Customer role | Sign-off only | Active partner | +| Risk exposure | High (late) | Low (early) | + +## Best For + + - Products with evolving requirements + - Startups, SaaS, consumer apps + +----- + +# 3\. Scrum + +## Overview + +Scrum is the most popular **Agile framework**. It imposes a specific +structure of roles, events, and artefacts on top of Agile principles. + +## Roles + +| Role | Responsibility | +| ----------------- | ------------------------------------------- | +| **Product Owner** | Owns the backlog; prioritises what to build | +| **Scrum Master** | Removes blockers; facilitates ceremonies | +| **Dev Team** | Self-organising; delivers the increment | + +## Artefacts + + - **Product Backlog** — ordered list of all desired features/fixes + - **Sprint Backlog** — subset of items pulled into the current sprint + - **Increment** — shippable product at end of each sprint + +## Ceremonies (Events) + +``` example +[Sprint Planning] → [Daily Standup × N] → [Sprint Review] → [Retrospective] + ↑_____________________________ repeat (1–4 weeks) ___________________| +``` + +| Event | Duration | Purpose | +| --------------- | -------- | ---------------------------------- | +| Sprint Planning | ≤ 8 hrs | Decide what goes into the sprint | +| Daily Standup | 15 min | Sync on progress, surface blockers | +| Sprint Review | ≤ 4 hrs | Demo increment to stakeholders | +| Retrospective | ≤ 3 hrs | Reflect and improve the process | + +## Example: Daily Standup Format + +``` example +Each team member answers three questions: + 1. What did I complete yesterday? + 2. What will I work on today? + 3. Is anything blocking me? +``` + +## Best For + + - Teams of 3–9 people + - Products with frequent reprioritisation + +----- + +# 4\. Kanban + +## Overview + +A **flow-based** method focused on visualising work and limiting +work-in-progress (WIP) to maximise throughput. No fixed sprints. + +## Core Principles + +1. Visualise the workflow (the Kanban board) +2. Limit WIP per column +3. Manage and improve flow +4. Make policies explicit + +## Example Board + +``` example +| Backlog | In Progress (WIP:2) | Review (WIP:1) | Done | +|---------|---------------------|----------------|------| +| Task D | Task B | Task A | Task X| +| Task E | Task C | | Task Y| +``` + +WIP limits prevent bottlenecks — new work cannot start until a slot opens. + +## Scrum vs Kanban + +| Dimension | Scrum | Kanban | +| --------- | ------------------- | ---------------------- | +| Cadence | Fixed sprints | Continuous flow | +| Roles | PO, SM, Dev Team | No prescribed roles | +| Change | After sprint | Anytime | +| Metrics | Velocity | Cycle time, throughput | +| Best for | Feature development | Operations, support | + +----- + +# 5\. Extreme Programming (XP) + +## Overview + +An Agile methodology with an extreme focus on **engineering practices** +and code quality. + +## Key Practices + + - **Test-Driven Development (TDD)**: Write the test before the code + - **Pair Programming**: Two developers work at one machine + - **Continuous Integration (CI)**: Merge and test code multiple times a day + - **Refactoring**: Continuously improve code structure + - **Small Releases**: Deploy frequently, in small increments + - **Collective Code Ownership**: Anyone can change any code at any time + +## Example: TDD Cycle + +``` example +Red → Write a failing test +Green → Write the minimum code to pass it +Refactor → Clean up without breaking the test +Repeat. +``` + +## Best For + + - Teams prioritising code quality and technical excellence + - Projects with rapidly changing requirements + +----- + +# 6\. SAFe (Scaled Agile Framework) + +## Overview + +SAFe scales Agile practices across **large enterprises** with multiple +teams working on the same product. + +## Key Concepts + + - **Agile Release Train (ART)**: A long-lived team of 50–125 people + - **Program Increment (PI)**: A 8–12 week planning cycle (like a "super sprint") + - **PI Planning**: All teams plan together face-to-face every PI + +## Levels + +``` example +Essential SAFe: Team + Program levels +Large Solution: + Solution Train (multiple ARTs) +Portfolio SAFe: + Portfolio strategy & funding +``` + +## Best For + + - Enterprises with 100+ engineers + - Organisations migrating from Waterfall to Agile at scale + +----- + +# 7\. DevOps (Methodology + Culture) + +## Overview + +DevOps bridges **development and operations** to enable faster, more +reliable software delivery through automation and collaboration. + +## Core Pillars + +| Pillar | Description | +| -------------------------- | -------------------------------------------- | +| **CI/CD** | Automate build, test, and deploy pipelines | +| **Infrastructure as Code** | Manage servers via code (Terraform, Ansible) | +| **Monitoring** | Observe systems in production continuously | +| **Blameless Culture** | Learn from failures without finger-pointing | + +## Example: CI/CD Pipeline + +``` example +Code Push → Build → Unit Tests → Integration Tests → Deploy to Staging + → Manual Approval → Deploy to Production → Monitor +``` + +## Best For + + - Any team wanting faster, safer releases + - Organisations running cloud-native infrastructure + +----- + +# Comparison Summary + +| Methodology | Structure | Delivery Cadence | Flexibility | Team Size | Best Fit | +| ----------- | ---------- | ---------------- | ----------- | --------- | ----------------------- | +| Waterfall | Sequential | End of project | Low | Any | Fixed scope, compliance | +| Agile | Iterative | Every sprint | High | Small–Med | Evolving requirements | +| Scrum | Iterative | Every 1–4 weeks | High | 3–9 | Feature development | +| Kanban | Flow-based | Continuous | Very High | Any | Ops, support, flow work | +| XP | Iterative | Weekly | High | Small | Quality-critical code | +| SAFe | Iterative | Every 8–12 weeks | Medium | 50–125+ | Enterprise Agile | +| DevOps | Continuous | On every commit | High | Any | Fast, reliable delivery | + +----- + +# Choosing the Right Methodology + +Ask these questions: + +1. **How stable are the requirements?** + - Stable → Waterfall / SAFe + - Changing → Agile / Scrum / Kanban +2. **How large is the team?** + - 3–9 → Scrum + - 50+ → SAFe + - Any size ops team → Kanban +3. **How important is engineering quality?** + - Very high → XP (TDD, pair programming) +4. **How often do you need to ship?** + - Daily → DevOps + CI/CD + - Weekly → Scrum / XP + - Continuously → Kanban +5. **What is the regulatory environment?** + - High compliance → Waterfall or hybrid + + + +``` example +Most modern teams use a HYBRID approach: + Scrum (planning cadence) + + Kanban board (visual workflow) + + XP practices (TDD, CI) + + DevOps (automated pipelines) +``` + +----- + +# Key Terminology Glossary + +| Term | Definition | +| ------------------ | ------------------------------------------------------------ | +| Sprint | A fixed time-box (1–4 wks) in Scrum for delivering work | +| Backlog | Prioritised list of work items | +| Velocity | Amount of work a team completes per sprint | +| WIP Limit | Maximum items allowed in a workflow stage simultaneously | +| CI/CD | Continuous Integration / Continuous Delivery (or Deployment) | +| TDD | Test-Driven Development | +| Retrospective | Meeting to inspect and improve team process | +| Definition of Done | Agreed criteria for when a task is truly "complete" | +| Epic | Large body of work broken into smaller user stories | +| User Story | Feature described from end-user perspective | diff --git a/Career/20260407102129-mvp_mvt.md b/Career/20260407102129-mvp_mvt.md new file mode 100644 index 0000000..e700991 --- /dev/null +++ b/Career/20260407102129-mvp_mvt.md @@ -0,0 +1,15 @@ +# MVP and MVT + + - An ****MVP**** is a ****minimal functional product**** built to validate what customers actually want by observing real usage. + - An ****MVT**** (often called ****Minimum Viable Experiment/Test****) is a ****small, fast, low‑cost test**** designed to validate a specific assumption before you build anything substantial. + +****Minimum Viable Product (MVP)**** +A simplified but working version of a product that early users can interact with. +Purpose: validate product‑market fit and gather real behavioural feedback. + +****Minimum Viable Test (MVT / MVE)**** +A quick experiment to validate a single assumption — often before building an MVP. +Examples: landing page, survey, fake‑door button, email test. + + - ****MVT**** = “Should we even build this?” + - ****MVP**** = “We think this is worth building — now let’s test the simplest working version.” diff --git a/Career/20260410115348-microlise_ess.md b/Career/20260410115348-microlise_ess.md new file mode 100644 index 0000000..2050f70 --- /dev/null +++ b/Career/20260410115348-microlise_ess.md @@ -0,0 +1,103 @@ +Joined team Shackleton on \<2026-04-07 Tue\>, who are now overseeing the deployment process for ESS applications. + +[ESP Deployment Pipeline — Index](id:a6c345df-8db9-4538-b87e-0e72e2414905) + +# AI generated overview of ESP + +## What's the Big Picture? + +Your team is building an **automated deployment pipeline** for a software suite called **ESP** (made by a company called ESS). Right now, deploying ESP to customer environments is done **manually** - someone has to go through a checklist and do things by hand. That's slow, error-prone, and relies on people who are leaving the company. The goal is to automate all of that. + +The pipeline will live in **Azure DevOps (AzDO)** - Microsoft's platform for CI/CD (building and deploying software automatically). + +## Why Is This Happening Now? + +The engineers who originally built and understood ESP have mostly left ESS. The ones who remain are tied up on paid customer work. So this task was handed off to a Microlise team (**Team Ludo**) who got the ball rolling - they got ESP *building* in AzDO and started on deployment. Now Ludo have been pulled onto other work too, and **your team** has inherited it. + +So you're the third team to touch this. Expect some rough edges and gaps in knowledge. + +## What Is ESP, Exactly? + +ESP is a **suite of applications** sold to customers. Think of it like a product bundle - each customer gets the apps they actually need (not every customer gets everything). It's delivered and runs on the customer's environment, which means: + + - You're deploying to **their servers**, not yours + - Different customers have different sets of apps installed + - Some customers might have **test and production on the same server** - which is a headache, because you have to be careful not to accidentally deploy to prod when you meant test + +The apps in the suite are: + + - **One database** - `EspBroker` (SQL Server database) + - **Six Windows Services** - background processes that run on a server (DLService, EmailService, etc.) + - **Three Citrix Applications** - desktop apps delivered via Citrix (VPlanner, VPlannerAdmin, ImportApp). Citrix is a technology that streams apps to users remotely, like a remote desktop but per-app. + +## What Is the Pipeline Actually Going to Do? + +The pipeline will run **PowerShell scripts** that walk through a deployment in stages. Think of it like a structured checklist that will eventually become fully automated. The stages are: + +| Stage | What it does | +| ----------------- | ---------------------------------------------------------------------------- | +| **Validate** | Loads and checks the manifest (config file describing the environment) | +| **Prerequisites** | Checks the target server has everything it needs before you touch it | +| **Predeploy** | Copies the new version's files to the server and configures them | +| **Deploy** | The actual upgrade - stops services, swaps old files for new, starts back up | +| **Postdeploy** | Checks everything came back up healthy | + +You can run any combination of these stages, so for example you might just run Validate to check your config is right, without actually deploying anything. + +## The Script Architecture (This Is Important) + +The scripts are designed in **three tiers**, like a layered cake: + +**Tier 1 - The Entry Point** You call one script (e.g. `Invoke-Deployment.ps1`) and tell it the customer, environment, and which stages to run. It loads the manifest, loads all the modules, then hands off to Tier 2. + +**Tier 2 - The Orchestrator** One script per stage (PreDeployment, Deployment, etc.). This is the "brain" - it decides *what* needs to happen and in what order, then calls Tier 3 to actually do it. It never does anything directly itself. + +**Tier 3 - The Workers** Small, self-contained functions that each do *one thing* - stop a Windows service, run a SQL query, copy a file, etc. They know nothing about ESP specifically. They just take parameters and do the job. This is useful because you can test them in isolation. + +**Why this structure matters for you:** Right now, many Tier 3 functions are essentially **placeholders** - instead of actually doing something, they prompt a human to do it manually. The plan is to replace those manual prompts with real automation over time. So initially the pipeline is more of a guided checklist than true automation. + +## The Manifest - What Is That? + +A manifest is a **config file** that describes a specific customer's environment - which apps they have, what servers they're on, credentials, etc. The idea is that you have one manifest per customer/environment combo (e.g.ROMAC DEV), and the scripts read that to know what to do. The manifest design isn't fully defined yet, which is one of the open items. + +## The Dacpac - What's That? + +A **dacpac** is a packaged SQL Server database schema. Instead of writing raw SQL migration scripts, you describe what the database *should look like*, and the dacpac tool works out what changes to make to get there. Team Ludo built one for the ESP database. + +**The big problem:** The existing customer databases are in an inconsistent state - they've drifted from the official schema over time (probably from manual fixes and patches). Before you can deploy via dacpac, someone will need to manually clean up each database to get it into a consistent state. Until that's done, the database deployment step can't be automated. + +## The Known Pain Points You Should Be Aware Of + +These are the things most likely to cause your team grief: + +1. **Inconsistent databases** - Can't use the dacpac until each customer's DB is manually fixed first. The pipeline needs to detect this and fail gracefully rather than making things worse. + +2. **The build artifact is a mess** - The ESP build produces a huge folder of DLLs all jumbled together, rather than neatly separated per application. Your scripts will need to figure out which files belong to which app. + +3. **No test environment** - You don't have a safe sandbox to practice deployments on. Any deployment you run is against a real customer environment. This is a significant risk. + +4. **Can't run locally** - Licensing restrictions mean you can't run ESP on your own machine to test things. Everything has to happen on the actual servers. + +5. **Citrix is complicated** - Deploying the Citrix apps requires notifying users, waiting for them to quit, killing sessions if they don't, and then doing the upgrade. The exact mechanism for the upgrade itself (how do you actually swap the Citrix app?) is still TBC. + +6. **Cycle checks** - Post-deployment checks involve navigating through the Citrix UI to verify things work. That's very hard to automate. + +## What's Still Not Decided + +Several important things are still open: + + - **Rollback plan** - If a deployment goes wrong, how do you undo it? Not defined yet. + - **Test plan** - How will you test the pipeline itself? + - **Citrix deployment method** - Believed to be a file copy but not confirmed + - **Manifest structure** - What does the config file actually look like? + - **Full scope** - Will this eventually cover production and ESS's own data centre too? + +## What Should You Focus On First? + +To provide real value quickly, I'd suggest getting comfortable with: + +1. **Azure DevOps YAML pipelines** - understand how they're structured and how they trigger scripts +2. **PowerShell scripting** - the whole deployment mechanism is PowerShell +3. **The existing Ludo work** - look at the existing pipeline runs and deploy scripts linked in the document before writing anything new +4. **The manifest concept** - helping define what that config file looks like is impactful foundational work +5. **The dacpac situation** - understanding which customer databases need manual cleanup before automation can work diff --git a/Career/20260410121527-ess_esp_index.md b/Career/20260410121527-ess_esp_index.md new file mode 100644 index 0000000..fd8968b --- /dev/null +++ b/Career/20260410121527-ess_esp_index.md @@ -0,0 +1,58 @@ +# Overview + +The ESP Deployment Pipeline project is an effort to automate the deployment of the ESP application suite (built by ESS) into customer environments using Azure DevOps (AzDO) YAML pipelines and PowerShell scripts. + +We are the third team to inherit this work. Team Ludo started it, got the build working in AzDO, and began deployment work before being reassigned. We pick up from there. + +# Quick Reference — Key Concepts + +| Concept | What it is | Notes File | +| --------------- | ---------------------------------------------------- | ------------------------------------------------------------------------------- | +| ESP | Suite of apps deployed per-customer | [ESP Applications](id:091e9bbc-0c9f-43da-81f9-464882c2a15b) | +| AzDO Pipeline | YAML-based CI/CD pipeline in Azure DevOps | [ESP — AzDO Pipeline](id:d66e946f-6785-4e8e-ba00-bb25a52237d1) | +| PowerShell Arch | Three-tier script architecture (T1/T2/T3) | [ESP — PowerShell Script Architecture](id:bf63b6c5-f32f-462e-82ad-8d4a15f7ba48) | +| Manifest | Config file defining a customer environment | [ESP — The Manifest](id:9bf19a8b-5581-4be8-9892-913c51df0128) | +| Dacpac | Packaged SQL Server DB schema deployment | [ESP — Database & Dacpac](id:46add22d-e562-4e3a-a301-d4aea2552952) | +| Stages | Validate / Prerequisites / Predeploy / Deploy / Post | [ESP — Deployment Stages](id:cd3cd02f-c9b3-4455-8ce5-b2a5aa458fed) | +| Known Issues | Pain points, risks, and gaps | [ESP — Known Issues & Risks](id:53d803aa-7e6f-44ed-8d99-a243a5ed5aab) | +| Open Questions | Things still TBC or not yet designed | [ESP — Open Questions & TBC Items](id:82c3d447-d6d3-498e-9f66-aee60c752462) | +| Glossary | Terminology reference | [ESP — Glossary](id:2c950d25-6f86-4f83-a7b4-69e2ca2f7023) | + +# Context — Why This Project Exists + + - Few original ESS engineers remain; those left are on paid customer work + - Manual deployments are slow, risky, and rely on tribal knowledge + - Business goal: *all deployments are consistently repeatable with no manual VM work* + - Initial scope: ROMAC DEV environment only + - Future scope: all environments including PROD and ESS's own data centre + +# Team History + +| Team | Contribution | Current Status | +| --------- | ------------------------------------------------------- | --------------------------- | +| ESS | Built ESP; wrote original deployment docs | Mostly departed | +| Team Ludo | Full ESP build in AzDO; started deploy pipeline; dacpac | Reassigned to customer work | +| Our Team | Inheriting deploy pipeline work | Active | + +# Existing Resources to Review + + - Handover from Team Ludo: `ESP_Pipeline_Handover.docx` + - Ludo deployment docs: `DEPLOY.md` (in Repos) + - Ludo deploy scripts: `deploy` folder (in Repos) + - ESP Main Build pipeline: AzDO → Pipelines → `ESS.esp Main Build` + - ESP Deploy Pipeline: AzDO → Pipelines → `ESS.esp Deploy` + +# File Index + +| File | Contents | +| ------------------------------------------------------------------------------- | --------------------------------------- | +| [ESP Deployment Pipeline — Index](id:a6c345df-8db9-4538-b87e-0e72e2414905) | This file — master index | +| [ESP Applications](id:091e9bbc-0c9f-43da-81f9-464882c2a15b) | ESP app suite breakdown | +| [ESP — AzDO Pipeline](id:d66e946f-6785-4e8e-ba00-bb25a52237d1) | AzDO pipeline structure and concepts | +| [ESP — PowerShell Script Architecture](id:bf63b6c5-f32f-462e-82ad-8d4a15f7ba48) | Three-tier PowerShell architecture | +| [ESP — The Manifest](id:9bf19a8b-5581-4be8-9892-913c51df0128) | Manifest design and purpose | +| [ESP — Database & Dacpac](id:46add22d-e562-4e3a-a301-d4aea2552952) | Database, dacpac, and SQL concerns | +| [ESP — Deployment Stages](id:cd3cd02f-c9b3-4455-8ce5-b2a5aa458fed) | Stages of deployment — detail per stage | +| [ESP — Known Issues & Risks](id:53d803aa-7e6f-44ed-8d99-a243a5ed5aab) | Known issues and risks | +| [ESP — Known Issues & Risks](id:53d803aa-7e6f-44ed-8d99-a243a5ed5aab) | TBC items and open design questions | +| [ESP — Glossary](id:2c950d25-6f86-4f83-a7b4-69e2ca2f7023) | Glossary of all technical terms | diff --git a/Career/20260410121719-esp_applications.md b/Career/20260410121719-esp_applications.md new file mode 100644 index 0000000..d1676e5 --- /dev/null +++ b/Career/20260410121719-esp_applications.md @@ -0,0 +1,90 @@ +# What is ESP? + +ESP is a software suite built by ESS and sold to customers. It is deployed on a **per-customer basis** — similar to TMC (another internal product) but with fewer applications in the suite. + +Key traits: + + - Customers only have the apps **they use** — not every customer gets everything + - Customers may have multiple environments (e.g. TEST and PROD) + - Test and PROD instances may live on the **same server** — this is a risk to be mindful of during deployment + +# Application Categories + +ESP applications fall into three categories: + +## 1\. Databases + +| Application | Type | Notes | +| ----------- | ---------- | ------------------------------------------- | +| EspBroker | SQL Server | The core ESP database. Deployed via dacpac. | + +See [ESP — Database & Dacpac](id:46add22d-e562-4e3a-a301-d4aea2552952) for dacpac detail. + +## 2\. Windows Services + +[Windows Services](id:faa7f193-5af6-4a2f-a73e-540f833a7fd0) are background processes that run on a Windows server. They have no UI — they start, run, and are managed via the Windows Service Manager (or PowerShell commands like `Stop-Service`, `Start-Service`). + +| Service Name | Notes | +| --------------- | --------------------------------- | +| DLService | Unknown specific function (TBC) | +| EmailService | Likely handles outbound emails | +| ExPlService | Unknown specific function (TBC) | +| MISService | Unknown specific function (TBC) | +| OutboundService | Likely handles outbound messaging | +| ULService | Unknown specific function (TBC) | + +### Working with Windows Services in PowerShell + +``` powershell +# Stop a service +Stop-Service -Name "DLService" -Force + +# Start a service +Start-Service -Name "DLService" + +# Check service status +Get-Service -Name "DLService" + +# Wait for a service to stop +(Get-Service -Name "DLService").WaitForStatus('Stopped', '00:01:00') +``` + +### Why Services Must Be Stopped Before Deployment + +When deploying a new version, the old service process holds file locks on its [DLLs](id:e717c252-0e15-4403-898f-93163dd1b147). You cannot overwrite a locked file on Windows. Therefore the sequence is: + +1. Stop the service +2. Swap the files (old → new) +3. Start the service + +## 3\. Citrix Applications + +Citrix is a technology that **streams desktop applications** to users remotely - the app runs on a server but the user sees it on their machine, similar to a remote desktop but on a per-app basis. + +| Application | Notes | +| ------------- | ---------------------------- | +| VPlanner | Main planning application | +| VPlannerAdmin | Admin interface for VPlanner | +| ImportApp | Data import application | + +### Citrix Deployment Considerations + +Citrix apps are more complex to deploy than Windows Services because: + + - **Active user sessions** may be running — you can't just swap files + - You must **notify users** of the upcoming upgrade and give a grace period + - After the grace period, **kill any remaining sessions** + - The actual upgrade mechanism is **still TBC** — believed to be a file copy with handling for locked executables built in + - Post-deploy **cycle checks** require navigating Citrix UI — hard to automate + +# Per-Customer App Lists + +Unlike TMC (where presumably all customers get everything), ESP is a subset deployment. This means: + + - The manifest (see [ESP — The Manifest](id:9bf19a8b-5581-4be8-9892-913c51df0128)) must define **which apps each customer has** + - The deployment scripts must skip apps not applicable to a given customer + - There is no universal "deploy everything" - each customer's list must be explicitly defined and maintained + +# Completeness of This List + +The handover document states this list is believed complete but **has not been confirmed**. Treat it as a working assumption, not a guarantee. Validate against actual customer environments when possible. diff --git a/Career/20260410121927-esp_pipeline.md b/Career/20260410121927-esp_pipeline.md new file mode 100644 index 0000000..d018547 --- /dev/null +++ b/Career/20260410121927-esp_pipeline.md @@ -0,0 +1,150 @@ +# What is Azure DevOps (AzDO)? + +Azure DevOps is Microsoft's platform for DevOps workflows. It covers: + + - **Repos** — Git source code repositories + - **Pipelines** — automated build and deployment (CI/CD) + - **Boards** — work items, epics, features, tasks + - **Artifacts** — storing build outputs (packages, DLLs, etc.) + +For this project, the relevant parts are **Pipelines** and **Repos**. + +# What is a YAML Pipeline? + +AzDO pipelines can be defined in two ways: through a GUI (classic) or via a YAML file stored in the repo. We use **YAML pipelines** — this means the pipeline definition lives in source control alongside the code, making it versioned and auditable. + +## Basic YAML Pipeline Structure + +``` yaml +trigger: + branches: + include: + - main + +pool: + vmImage: 'windows-latest' # or a self-hosted agent + +stages: + - stage: Build + jobs: + - job: BuildJob + steps: + - task: PowerShell@2 + inputs: + filePath: 'scripts/Invoke-Deployment.ps1' + arguments: '-Customer ROMAC -Environment DEV -Stages PREDEPLOY,DEPLOY' +``` + +## Key YAML Concepts + +| Term | Meaning | +| ---------- | ------------------------------------------------------------------- | +| `trigger` | What causes the pipeline to run (e.g. a push to main) | +| `pool` | The agent (machine) that runs the pipeline | +| `stage` | A high-level grouping of jobs (e.g. Build, Deploy) | +| `job` | A unit of work that runs on one agent | +| `step` | An individual action within a job (run a script, call a task, etc.) | +| `task` | A pre-built step from the AzDO marketplace (e.g. PowerShell@2) | +| `artifact` | Output from one stage/job that can be consumed by another | + +# The ESP Pipelines + +There are two existing pipelines to be aware of: + +## ESS.esp Main Build + + - **Purpose:** Builds the ESP applications from source code + - **Output:** Build artifacts (DLLs, binaries) + - **Known Issue:** The artifact does not separate applications cleanly - it produces a large flat folder of DLLs. See [ESP — Known Issues & Risks](id:53d803aa-7e6f-44ed-8d99-a243a5ed5aab) + - **Location:** AzDO → Pipelines → Runs for `ESS.esp Main Build` + +## ESS.esp Deploy + + - **Purpose:** Deploys ESP to a customer environment + - **Status:** Started by Team Ludo, incomplete + - **Location:** AzDO → Pipelines → Runs for `ESS.esp Deploy` + - This is the primary pipeline your team is responsible for completing + +# How the Pipeline Calls Our Scripts + +The pipeline's job is relatively thin - it is an **orchestrator** that calls our PowerShell entry point with the correct parameters. + + Pipeline (YAML) + └── Calls: Invoke-Deployment.ps1 -Customer ROMAC -Environment DEV -Stages PREDEPLOY,DEPLOY,POSTDEPLOY + └── Tier 1 script (loads manifest, loads modules) + └── Calls Tier 2 scripts (PreDeployment.ps1, Deployment.ps1, etc.) + └── Calls Tier 3 functions (Stop-WindowsService, Invoke-Sql, etc.) + +See [ESP — PowerShell Script Architecture](id:bf63b6c5-f32f-462e-82ad-8d4a15f7ba48) for full detail on the script architecture. + +# Pipeline Agents + +An **agent** is the machine that actually runs the pipeline. There are two types: + +| Type | Description | +| ---------------- | ----------------------------------------------------------- | +| Microsoft-hosted | Azure spins up a fresh VM for each run; ephemeral | +| Self-hosted | A persistent machine you manage; has access to your network | + +For deploying to customer VMs (ROMAC DEV), we will almost certainly need a **self-hosted agent** — a Microsoft-hosted agent running in Azure would not have network access to the customer's internal servers. + +# Pipeline Variables and Secrets + +Pipelines can use variables for configuration. Sensitive values (passwords, connection strings) should be stored as **secret variables** or in **Azure Key Vault**, never hardcoded in the YAML. + +``` yaml +variables: + - name: CustomerName + value: ROMAC + - name: DbPassword + value: $(DB_PASSWORD) # references a secret variable set in AzDO UI +``` + +# Approvals and Gates + +For higher environments (PROD), AzDO supports **manual approval gates** between stages — a human must approve before the pipeline continues. This is important when the scope expands beyond DEV. + +# Pipeline Run History + +Each pipeline run is logged in AzDO with full step-by-step output. This is your primary debugging tool when a deployment fails. + +# Relationship to Our PowerShell Scripts + +A key design goal is that the PowerShell scripts should be able to run **independently of AzDO** — i.e. you could call `Invoke-Deployment.ps1` directly from a terminal if needed. AzDO is just a convenient trigger and logging wrapper. This means the logic must not be baked into the YAML itself. + +# One other note on agents: + +Almost certainly ****Virtual Machines (VMs)****. Physical ("bare metal") servers are rarely used in modern infrastructure for this kind of thing. A data centre typically runs a small number of powerful physical machines, and on top of those you run many VMs — each VM behaves like its own independent server but they're all sharing the underlying physical hardware. + +So the full picture is probably: + +``` +Physical machine(s) in ESS Altrincham DC +└── VM: AzDO Self-Hosted Agent +└── VM: App Server (runs the Windows Services, Citrix apps) +└── VM: SQL Server (runs EspBroker database) +... possibly more VMs + +``` + +And the flow when you kick off a deployment in AzDO: + +``` + +You click "Run Pipeline" in AzDO (cloud) + │ + ▼ +AzDO talks to the self-hosted agent VM in Altrincham + │ + ▼ +The agent picks up the job and runs the PowerShell scripts + │ + ▼ +The scripts connect (via PowerShell remoting) to the App Server VM +The scripts connect to the SQL Server VM + │ + ▼ +Deployment happens +``` + +The agent itself isn't the thing being deployed **to** — it's just the middleman that has network access to the other VMs in the same DC. Which also ties back to why each customer needs unique credentials — even though they share that infrastructure in DEV, each customer's VMs are in their own domain, so the agent needs the right credentials to authenticate into each one. diff --git a/Career/20260410122116-ess_scripts.md b/Career/20260410122116-ess_scripts.md new file mode 100644 index 0000000..af8235b --- /dev/null +++ b/Career/20260410122116-ess_scripts.md @@ -0,0 +1,270 @@ +# Overview + +The deployment scripts are structured in **three tiers**, loosely analogous to a presentation/domain/data layered architecture in software. This separation exists to: + + - Keep ESP-specific knowledge isolated to higher tiers + - Allow Tier 3 functions to be tested in isolation + - Allow the scripts to run independently of AzDO + - Make incremental automation easier (replace manual prompts one function at a time) + +# Entry Point — How to Call the Scripts + +``` powershell +Invoke-Deployment.ps1 -Customer ROMAC -Environment DEV -Stages PREDEPLOY,DEPLOY,POSTDEPLOY +``` + +You can pass any combination of stages. For example, to only validate: + +``` powershell +Invoke-Deployment.ps1 -Customer ROMAC -Environment DEV -Stages VALIDATE +``` + +# Tier 1 — Entry Point (Presentation Layer) + +## Responsibility + + - The **single entry point** to the whole deployment system + - Loads and validates the manifest (see [ESP — The Manifest](id:9bf19a8b-5581-4be8-9892-913c51df0128)) + - Loads all required PowerShell modules (Tier 2, Tier 3, helpers) + - Calls the appropriate Tier 2 functions for the requested stages + - Passes the manifest object through to Tier 2 + +## Key Characteristics + + - One script: `Invoke-Deployment.ps1` + - Does NOT contain deployment logic itself + - Acts as a wiring layer only + +## Pseudocode + +``` powershell +param( + [string]$Customer, + [string]$Environment, + [string[]]$Stages +) + +$manifest = Load-Manifest -Customer $Customer -Environment $Environment +Assert-ManifestValid -Manifest $manifest + +Import-Module ./Tier2/PreDeployment.psm1 +Import-Module ./Tier2/Deployment.psm1 +Import-Module ./Tier2/PostDeployment.psm1 +Import-Module ./Helpers/Logging.psm1 + +foreach ($stage in $Stages) { + switch ($stage) { + "PREDEPLOY" { Invoke-PreDeployment -Manifest $manifest } + "DEPLOY" { Invoke-Deployment -Manifest $manifest } + "POSTDEPLOY" { Invoke-PostDeployment -Manifest $manifest } + } +} +``` + +# Tier 2 — Orchestration Layer (Domain Layer) + +## Responsibility + + - One script per deployment stage (e.g. `PreDeployment.psm1`, `Deployment.psm1`) + - Contains the **logic and sequencing** of what needs to happen + - Decides **which** Tier 3 functions to call and in what order + - Reacts to return values from Tier 3 (e.g. if a function fails, abort or retry) + +## Key Constraints + + - **Does NOT** pass the manifest to Tier 3 — all required data is extracted and passed as explicit parameters + - **Does NOT** directly perform any action itself — delegates entirely to Tier 3 + - **Does** have knowledge of ESP and what a deployment involves + +## Why "Does Not Pass Manifest to Tier 3"? + +Tier 3 functions are designed to be generic and reusable. If they accepted a manifest object, they'd need to know about its structure — breaking isolation. Instead, Tier 2 extracts what Tier 3 needs: + +``` powershell +# BAD — passes the whole manifest (couples Tier 3 to manifest structure) +Stop-WindowsService -Manifest $manifest + +# GOOD — extracts what's needed and passes explicitly +Stop-WindowsService -ServiceName $manifest.Services.DLService.Name ` + -ServerName $manifest.Servers.AppServer +``` + +## Scripts in This Tier + +| Script | Stage it covers | +| --------------------- | ------------------- | +| `Validation.psm1` | VALIDATE stage | +| `Prerequisites.psm1` | PREREQUISITES stage | +| `PreDeployment.psm1` | PREDEPLOY stage | +| `Deployment.psm1` | DEPLOY stage | +| `PostDeployment.psm1` | POSTDEPLOY stage | + +# Tier 3 — Functional Layer (Workers) + +## Responsibility + + - Small, **single-purpose** functions + - No knowledge of ESP, the manifest, or deployment context + - Accept all required data as **explicit parameters** + - Can be tested in isolation with mock data + +## Key Characteristics + + - Examples: `Stop-WindowsService`, `Invoke-SqlScript`, `Copy-Files`, `Get-RemoteSession`, `Send-CitrixNotification` + - Initially implemented as **manual prompt wrappers** — the function prints instructions to a human and waits for confirmation + - Will be replaced with real automation incrementally + +## The Manual Prompt Pattern (Current State) + +``` powershell +function Stop-WindowsService { + param([string]$ServiceName, [string]$ServerName) + + # Current implementation — prompts a human + $response = Invoke-ManualPrompt -Message "Please stop service '$ServiceName' on '$ServerName', then press Enter." + return $response +} +``` + +## Target Implementation (Automated) + +``` powershell +function Stop-WindowsService { + param([string]$ServiceName, [string]$ServerName) + + $session = Get-RemoteSession -ServerName $ServerName + Invoke-Command -Session $session -ScriptBlock { + Stop-Service -Name $using:ServiceName -Force + (Get-Service -Name $using:ServiceName).WaitForStatus('Stopped', '00:01:00') + } +} +``` + +## Exceptions — Functions That Cannot Be Manual Prompts + +A small number of Tier 3 functions **must** be implemented for real from the start because they return data the scripts need to function: + +| Function | Why it can't be a manual prompt | +| ------------------- | ----------------------------------------------------- | +| `Load-Manifest` | Returns the manifest object — must actually read file | +| `Get-RemoteSession` | Returns a PS session — must actually connect | +| `Read-File` | Returns file contents — must actually read | + +# Helper Modules + +In addition to the three tiers, **general-purpose helper modules** exist that can be called from anywhere (Tier 1, 2, or 3): + +| Module | Purpose | +| -------------- | ------------------------------------------ | +| `Logging.psm1` | Write structured log output | +| `Prompt.psm1` | The manual prompt mechanism used by Tier 3 | +| | | + +Helpers follow the same rule as Tier 3: **the manifest is never passed to them**. + +# Testing Strategy + +Because Tier 3 functions are isolated, they can be unit tested without any connection to a real ESP environment: + +``` powershell +# Example Pester test for Stop-WindowsService +Describe "Stop-WindowsService" { + It "calls Invoke-Command with correct service name" { + Mock Invoke-Command {} + Mock Get-RemoteSession { return [PSCustomObject]@{ Session = "MockSession" } } + + Stop-WindowsService -ServiceName "DLService" -ServerName "SERVER01" + + Assert-MockCalled Invoke-Command -Times 1 + } +} +``` + +# Incremental Automation Plan + +The architecture is designed so that automation can be added **one function at a time**, without restructuring anything: + +1. Deploy with all Tier 3 functions as manual prompts (guided checklist) +2. Identify the lowest-risk, simplest functions to automate first +3. Replace manual prompts with real implementations one by one +4. Each replacement can be independently tested before deployment + +Suggested automation order (rough): + +1. `Stop-WindowsService` / `Start-WindowsService` — well-understood, low risk +2. `Copy-Files` — straightforward file operations +3. `Invoke-SqlScript` — once databases are in dacpac-ready state +4. Citrix-related functions — last, most complex + +# Architecture Diagram (Text) + + AzDO Pipeline (YAML) + │ + └─► Invoke-Deployment.ps1 [TIER 1] + │ Loads manifest + │ Loads all modules + │ + ├─► PreDeployment.psm1 [TIER 2] + │ ├─► Copy-Files [TIER 3] + │ └─► ... + │ + ├─► Deployment.psm1 [TIER 2] + │ ├─► Stop-WindowsService [TIER 3] + │ ├─► Invoke-SqlScript [TIER 3] + │ ├─► Start-WindowsService [TIER 3] + │ └─► ... + │ + └─► PostDeployment.psm1 [TIER 2] + ├─► Get-ServiceStatus [TIER 3] + └─► ... + + [Helpers: Logging, Prompt — available at any tier] + +## Mermaid Diagram (Text) + +``` mermaid +graph TD + + %% Tier 1 + subgraph "Tier 1 - Entry Point" + A[Invoke-Deployment.ps1] + end + + %% Tier 2 + subgraph "Tier 2 - Deployment Phases" + B[PreDeployment.psm1] + C[Deployment.psm1] + D[PostDeployment.psm1] + end + + %% Tier 3 + subgraph "Tier 3 - Actions" + E[Copy-Files] + F[Stop-WindowsService] + G[Invoke-SqlScript] + H[Start-WindowsService] + I[Get-ServiceStatus] + end + + %% Helpers + subgraph "Helpers (All Tiers)" + J[Logging Helper] + K[Prompt Helper] + end + + %% Relationships + A --> B + A --> C + A --> D + + B --> E + + C --> F + C --> G + C --> H + + D --> I + + A --> J + A --> K +``` diff --git a/Career/20260410122303-ess_manifest.md b/Career/20260410122303-ess_manifest.md new file mode 100644 index 0000000..5898c1c --- /dev/null +++ b/Career/20260410122303-ess_manifest.md @@ -0,0 +1,118 @@ +# What Is the Manifest? + +The manifest is a **configuration file** that describes a specific customer environment. The deployment scripts read the manifest to understand **what** to deploy, **where**, and **how**. + +There will be one manifest per customer/environment combination, for example: + + - `ROMAC_DEV.json` (or `.yaml`, `.psd1` — format TBC) + - `ROMAC_PROD.json` + - `ACMECORP_DEV.json` + +# Why the Manifest Matters + +Without a manifest, the scripts would have no way of knowing: + + - Which server(s) to connect to + - Which apps this customer actually uses + - What credentials to use + - What version is being deployed + - What environment-specific configuration to apply + +It is the **single source of truth** for a deployment. Tier 1 loads it, validates it, and passes it to Tier 2. Tier 2 extracts values from it and passes those to Tier 3. + +# Design Status + +The manifest **structure has not yet been fully designed**. This is an open work item. See [ESP — Open Questions & TBC Items](id:82c3d447-d6d3-498e-9f66-aee60c752462) for a list of design questions to resolve. + +What follows is a **proposed structure** based on what the scripts will need. + +# Proposed Manifest Structure + +``` json +{ + "customer": "ROMAC", + "environment": "DEV", + "version": "3.2.1", + + "servers": { + "appServer": "ROMAC-DEV-APP01", + "dbServer": "ROMAC-DEV-DB01", + "citrixServer": "ROMAC-DEV-CTX01" + }, + + "database": { + "name": "EspBroker", + "dacpacReady": false + }, + + "windowsServices": { + "DLService": { "enabled": true, "installPath": "C:\\ESP\\DLService" }, + "EmailService": { "enabled": true, "installPath": "C:\\ESP\\EmailService" }, + "ExPlService": { "enabled": false }, + "MISService": { "enabled": true, "installPath": "C:\\ESP\\MISService" }, + "OutboundService": { "enabled": true, "installPath": "C:\\ESP\\OutboundService" }, + "ULService": { "enabled": false } + }, + + "citrixApps": { + "VPlanner": { "enabled": true, "installPath": "C:\\ESP\\VPlanner" }, + "VPlannerAdmin": { "enabled": true, "installPath": "C:\\ESP\\VPlannerAdmin" }, + "ImportApp": { "enabled": false } + }, + + "citrix": { + "notificationMinutes": 15, + "sessionKillGracePeriodMinutes": 5 + } +} +``` + +# Key Fields to Define + +| Field | Purpose | +| --------------------------- | ----------------------------------------------------------- | +| `customer` | Identifies the customer | +| `environment` | Identifies the environment (DEV/TEST/PROD) | +| `version` | Version of ESP being deployed | +| `servers.*` | Hostnames/IPs of servers to connect to | +| `database.dacpacReady` | Flag to indicate if DB is in a state for dacpac deployment | +| `windowsServices.*.enabled` | Whether this customer uses this service | +| `citrixApps.*.enabled` | Whether this customer uses this Citrix app | +| `citrix.*` | Citrix-specific config (notification timing, grace periods) | + +# Manifest Validation (VALIDATE Stage) + +The VALIDATE stage (see [ESP — Deployment Stages](id:cd3cd02f-c9b3-4455-8ce5-b2a5aa458fed)) loads the manifest and checks it is well-formed before any deployment activity. Things to validate: + + - All required fields are present + - Server names are resolvable/pingable + - `version` matches available build artifacts + - `dacpacReady` flag prevents DB deployment if false + - No conflicting settings (e.g. test and prod on same server without flag) + +# Manifest Location and Storage + +Where manifests should live is TBC, but candidates include: + + - A dedicated folder in the AzDO repo (versioned with the scripts) + - A separate config repo + - An external config store (Azure App Configuration, Key Vault, etc.) + +Sensitive values (passwords, connection strings) should **never** be in the manifest file in plaintext — they should reference a secret store. + +# Per-Customer App Lists + +Because not all customers use all apps, the manifest is the mechanism that drives per-customer deployment. Tier 2 scripts iterate over enabled apps: + +``` powershell +foreach ($service in $manifest.WindowsServices.GetEnumerator()) { + if ($service.Value.Enabled) { + Stop-WindowsService -ServiceName $service.Key ` + -ServerName $manifest.Servers.AppServer + } +} +``` + +# Environments on the Same Server + +The handover document flags that some customers may have TEST and PROD on the same physical server. The manifest must account for this — likely via distinct install paths per environment and explicit environment tagging to prevent accidental cross-environment deployment. diff --git a/Career/20260410122425-ess_database.md b/Career/20260410122425-ess_database.md new file mode 100644 index 0000000..34c1bd7 --- /dev/null +++ b/Career/20260410122425-ess_database.md @@ -0,0 +1,126 @@ +# The ESP Database + +ESP uses a single SQL Server database called **EspBroker**. This is deployed and upgraded as part of the overall ESP deployment process. + +# What Is a Dacpac? + +A **dacpac** (Data-tier Application Package) is a packaged representation of a SQL Server database **schema**. It is a file with the extension `.dacpac`. + +Rather than writing migration scripts that say "ALTER TABLE, ADD COLUMN…" manually, you instead describe the **desired end state** of the database, and the dacpac deployment tool (`sqlpackage`) calculates the delta and applies it automatically. + +## How Dacpac Deployment Works + + Your dacpac file (desired schema) + │ + ▼ + sqlpackage.exe + │ + ├── Connects to target database + ├── Compares desired schema to actual schema + ├── Generates a diff + └── Applies the diff (ALTER TABLE, CREATE INDEX, etc.) + +## Advantages Over Raw SQL Scripts + +| Dacpac | Raw SQL Scripts | +| --------------------------------------- | ----------------------------------------- | +| Declarative — describe what you want | Imperative — describe each change step | +| Tool calculates the diff automatically | You must track and apply changes manually | +| Idempotent — safe to run multiple times | Can fail if run twice without care | +| Schema is versioned as code | Scripts can get out of sync | + +## Relevant PowerShell / CLI + +``` powershell +# Deploy a dacpac using sqlpackage +& "C:\Program Files\Microsoft SQL Server\160\DAC\bin\sqlpackage.exe" ` + /Action:Publish ` + /SourceFile:"EspBroker.dacpac" ` + /TargetServerName:"ROMAC-DEV-DB01" ` + /TargetDatabaseName:"EspBroker" +``` + +# Team Ludo's Dacpac + +Team Ludo built a dacpac solution for EspBroker. This is already in the repo and represents the **target schema** that all customer databases should eventually conform to. + +# The Critical Problem — Schema Inconsistency + +## What the Problem Is + +Existing customer databases have drifted from the official schema over time. This drift likely happened due to: + + - Manual hotfixes applied directly to production databases + - Different versions of ESP deployed to different customers at different times + - No enforced schema management historically + +This means the dacpac's expected schema does not match what is actually in customer databases. + +## Consequence + +If you attempt to deploy the dacpac against an inconsistent database, `sqlpackage` will either: + + - Fail with errors (best case — nothing is changed) + - Apply incorrect changes that corrupt data (worst case) + +## What Needs to Happen + +Before the dacpac can be used for a customer: + +1. A database expert must **manually inspect** the customer's database +2. Identify all schema differences between actual and expected state +3. Write and apply **manual SQL scripts** to bring the DB into alignment +4. Verify the dacpac can then deploy cleanly (ideally against a copy) +5. Only then mark the customer as `dacpacReady: true` in the manifest + +## Pipeline Safeguard + +The deployment pipeline **must** check the `dacpacReady` flag before attempting database deployment, and fail clearly if it is `false`: + +``` powershell +function Invoke-DatabaseDeployment { + param($Manifest) + + if (-not $Manifest.Database.DacpacReady) { + Write-Error "Database for $($Manifest.Customer) $($Manifest.Environment) is not dacpac-ready. " + + "Manual schema alignment is required before deployment." + throw "Database not ready for dacpac deployment." + } + + # Proceed with dacpac deployment... + Deploy-Dacpac -DacpacPath $artifactPath -Server $Manifest.Servers.DbServer -Database "EspBroker" +} +``` + +# Database Deployment in the Deployment Stage + +Database deployment sits inside the **DEPLOY stage**. See [ESP — Deployment Stages](id:cd3cd02f-c9b3-4455-8ce5-b2a5aa458fed) for full stage detail. The database is typically deployed before services are started up to ensure the schema is ready for the new application code. + +Suggested order within the DEPLOY stage: + +1. Stop Windows Services +2. Deploy database dacpac (if dacpacReady) +3. Handle Citrix sessions +4. Swap application files +5. Start Windows Services back up + +# Rollback Considerations + +If the dacpac deploys successfully but a subsequent step fails, rolling back the database is non-trivial. Dacpac does not natively support rollback — options are: + + - Restore from backup (requires a backup to have been taken immediately before) + - Write a counter-dacpac that reverts the schema (complex, error-prone) + +This is one of the reasons the **rollback plan is still TBC**. See [ESP — Open Questions & TBC Items](id:82c3d447-d6d3-498e-9f66-aee60c752462). + +A **database backup must be taken before any deployment** — this should be a mandatory step in the PREDEPLOY stage. + +# SQL Server Concepts Relevant Here + +| Concept | Relevance | +| -------------- | ------------------------------------------------------------ | +| Schema | The structure of tables, columns, indexes, constraints, etc. | +| `sqlpackage` | Microsoft CLI tool that applies dacpac files | +| `BACPAC` | Like dacpac but includes data — useful for backup/restore | +| SQL Agent Jobs | Scheduled SQL jobs that may need to be handled during deploy | +| Linked Servers | DB connections to other servers — may be part of ESP's setup | diff --git a/Career/20260410122604-ess_deploy_stages.md b/Career/20260410122604-ess_deploy_stages.md new file mode 100644 index 0000000..c05c026 --- /dev/null +++ b/Career/20260410122604-ess_deploy_stages.md @@ -0,0 +1,161 @@ +# Overview + +The deployment is broken into **discrete stages** that can be run in any combination. This allows you to, for example, only run validation, or only run post-deploy checks, without triggering a full deployment. + +Entry point call: + +``` powershell +Invoke-Deployment.ps1 -Customer ROMAC -Environment DEV -Stages VALIDATE,PREDEPLOY,DEPLOY,POSTDEPLOY +``` + +# Stage Summary + +| Stage | Script (Tier 2) | Safe to Run Alone? | Notes | +| ------------- | --------------------- | ------------------ | ------------------------- | +| VALIDATE | `Validation.psm1` | Yes — read only | No changes made | +| PREREQUISITES | `Prerequisites.psm1` | Yes — read only | No changes made | +| PREDEPLOY | `PreDeployment.psm1` | Yes (with care) | Stages files, no swap yet | +| DEPLOY | `Deployment.psm1` | No — destructive | Requires PREDEPLOY first | +| POSTDEPLOY | `PostDeployment.psm1` | Yes (after DEPLOY) | Checks only, no changes | + +# Stage 1 — VALIDATE + +## Purpose + +Load the manifest and verify it is well-formed and consistent before anything else happens. + +## What It Does + + - Reads the manifest file for the given customer/environment + - Checks all required fields are present + - Validates server names, version references, and flags + - Checks that the referenced build artifact version exists + - Fails fast with a clear error if anything is wrong + +## What It Does NOT Do + + - Makes no changes to any server + - Does not connect to target servers (it may ping/resolve names only) + +## Why It Exists Separately + +You might want to validate a new manifest you've written without triggering a full deployment. Running VALIDATE alone takes seconds and gives you confidence before committing to the rest. + +# Stage 2 — PREREQUISITES + +## Purpose + +Verify that the target environment meets all requirements before deployment begins. + +## What It Checks (Not Exhaustive — Full List TBC) + + - Required software is installed on target servers (e.g. .NET runtime, SQL client) + - Correct environment labels/flags are set up + - Services exist (so the deployment can stop/start them) + - Network connectivity between agent and target servers + - Sufficient disk space on target servers + - Database is accessible + - If DEPLOY will include DB: check `dacpacReady` flag (see [ESP — Database & Dacpac](id:46add22d-e562-4e3a-a301-d4aea2552952)) + +## Outcome + + - Pass: continue to next stage + - Fail: abort with a clear list of what's missing — **no changes made** + +# Stage 3 — PREDEPLOY + +## Purpose + +Prepare the new version's files on the target servers **without** switching anything live. Think of it as laying everything out ready before the actual switch. + +## What It Does + + - Copies build artifacts from the AzDO artifact store to a **staging location** on each target server (not the live installation path) + - Applies environment-specific configuration to the staged files (e.g. replaces connection strings, config values from the manifest) + - Takes a **backup of the current live installation** and the database + - Verifies the staged files look correct + +## Why Stage Before Deploying? + +Pre-staging minimises the time during which the application is down. When DEPLOY runs, it can simply swap directories (fast) rather than copying large amounts of data (slow) while services are stopped. + +## Artifact Staging Note + +The ESP build artifact is currently a **flat folder of DLLs** — not separated per application. Predeploy will need to handle the mapping of DLLs to their correct applications. See [ESP — Known Issues & Risks](id:53d803aa-7e6f-44ed-8d99-a243a5ed5aab). + +# Stage 4 — DEPLOY + +## Purpose + +The actual upgrade — swap the old version for the new version. + +## Sequence (High Level) + +### 4a. Database + +1. Check `dacpacReady` flag — fail if false +2. Deploy the dacpac to EspBroker (see [ESP — Database & Dacpac](id:46add22d-e562-4e3a-a301-d4aea2552952)) + +### 4b. Windows Services + +1. Stop all enabled Windows Services (one by one or in parallel — TBC) +2. For each service: + a. Move/delete old installation directory + b. Move staged new files into the installation directory +3. Apply any final configuration adjustments + +### 4c. Citrix Applications + +1. Send notification to active Citrix users (e.g. "System upgrade in 15 mins") +2. Wait for grace period to expire +3. Kill any remaining Citrix sessions +4. Perform Citrix application upgrade (mechanism TBC — believed to be file copy) + +### 4d. Start Up + +1. Start all enabled Windows Services +2. Wait for each to reach Running state before moving on + +## Order Matters + +The database schema must be deployed **before** starting the new application code, because the new application version expects the new schema. Starting services before the dacpac is deployed would likely cause errors. + +## Risk + +This is the most **destructive** stage. If something goes wrong mid-deploy, the system may be in a partially upgraded state. This is why rollback planning (currently TBC) is critical. See [ESP — Open Questions & TBC Items](id:82c3d447-d6d3-498e-9f66-aee60c752462). + +# Stage 5 — POSTDEPLOY + +## Purpose + +Verify the deployment succeeded and the system is healthy. + +## What It Does + +### Basic Service Checks + + - Verify all enabled Windows Services are in `Running` state + - Verify no services crashed immediately after start + +### Cycle Checks + + - Extensive checks that exercise the application's functionality + - Currently heavily reliant on **Citrix UI navigation** — very difficult to automate + - May initially be a manual prompt ("Please run the cycle checks and confirm pass/fail") + - Long term: UI automation or API-based checks + +## What It Does NOT Do + + - Makes no changes to the system + - Should be safe to re-run at any time + +# Running Subsets of Stages + +| Goal | Stages to Run | +| --------------------------------- | ---------------------------------- | +| Check config before deploying | `VALIDATE` | +| Full pre-flight check | `VALIDATE,PREREQUISITES` | +| Stage files only (no deploy yet) | `VALIDATE,PREREQUISITES,PREDEPLOY` | +| Full deployment | All stages | +| Re-run health checks after deploy | `POSTDEPLOY` | +| Everything except DB | Custom flags TBC | diff --git a/Career/20260410122754-esp_known_issues.md b/Career/20260410122754-esp_known_issues.md new file mode 100644 index 0000000..4913e51 --- /dev/null +++ b/Career/20260410122754-esp_known_issues.md @@ -0,0 +1,161 @@ +# Overview + +This file documents all known issues, risks, and blockers identified in the handover material. Understanding these early will help you avoid surprises. + +# Issue 1 — Database Schema Inconsistency + +## What the Problem Is + +Existing customer databases have drifted from the official ESP schema. The dacpac built by Team Ludo reflects what the schema **should** be, but what's actually in customer databases doesn't match. + +## Impact + + - The dacpac **cannot be used** against any customer database until that database is manually aligned + - Deploying a dacpac against a mismatched DB risks data corruption or failures + +## What Needs to Happen + +For each customer database: + +1. A DBA or developer inspects the actual DB schema +2. Identifies differences from the dacpac's expected schema +3. Writes and applies manual scripts to bring the DB into line +4. Verifies a trial dacpac deployment succeeds against a copy +5. Marks the manifest `dacpacReady: true` + +## Mitigations in the Pipeline + + - The pipeline **must** check the `dacpacReady` flag before touching the database + - If `false`, fail immediately with a clear error message + - See [ESP — Database & Dacpac](id:46add22d-e562-4e3a-a301-d4aea2552952) for implementation detail + +# Issue 2 — Build Artifact Is a Flat DLL Dump + +## What the Problem Is + +The ESP Main Build pipeline produces a single large flat folder containing all DLLs for all applications mixed together. There is no clear separation like: + + /artifacts/DLService/DLService.dll + /artifacts/EmailService/EmailService.dll + +Instead it's more like: + + /artifacts/DLService.dll + /artifacts/SomeSharedLibrary.dll + /artifacts/EmailService.dll + /artifacts/AnotherThing.dll + ... (hundreds of files) + +## Impact + + - The PREDEPLOY stage can't simply copy a folder to a service's install path + - Scripts will need to know **which DLLs belong to which application** + - This mapping does not yet exist + +## What Needs to Happen + + - Investigate the build artifact structure in detail + - Create a mapping: application → list of DLLs/files it needs + - This mapping may need to be stored in the scripts, the manifest, or a separate config file + - Ideally, feed back to the build team to separate the artifacts properly + +# Issue 3 — No Test Environment + +## What the Problem Is + +There is no dedicated sandbox environment to run test deployments against. +Any deployment the team runs is against a **real customer environment**. + +## Impact + + - Mistakes affect real customers + - You cannot safely iterate and test the pipeline without risk + - Developing and debugging deployment logic is much harder + +## What Needs to Happen + + - Set up a dedicated ESP test environment (raised as TBC in handover) + - This may require ESS to provision a server, or Microlise to build one + - Until then, exercise **extreme caution** with any deployment run + +# Issue 4 — Licensing Prevents Local Execution + +## What the Problem Is + +ESP cannot be run on developer machines due to licensing restrictions. + +## Impact + + - You cannot test your deployment scripts against a local ESP instance + - Debugging requires deploying to a real (or test) server + - Makes the development feedback loop longer + +## Mitigation + + - The three-tier architecture helps here: Tier 3 functions can be unit tested in isolation without a real ESP environment + - Use Pester (PowerShell testing framework) for unit tests + - End-to-end testing requires access to a real environment + +# Issue 5 — Cycle Checks Require Citrix UI Navigation + +## What the Problem Is + +Post-deployment validation ("cycle checks") involves navigating through the Citrix application UI to verify functionality. This is: + + - Extensive and time-consuming + - Very hard to automate (requires UI automation tooling like Selenium or Tosca) + - Dependent on Citrix being accessible from the test runner + +## Short-Term Mitigation + +Implement cycle checks as a manual prompt in the POSTDEPLOY stage — the pipeline pauses and waits for a human to confirm the checks passed. + +## Long-Term Options + + - Investigate API-level checks if ESP exposes any (bypasses UI entirely) + - UI automation tools (e.g. Ranorex, Tosca, Selenium with Citrix plugin) + - Simplified smoke tests that don't require full UI navigation + +# Issue 6 — Citrix Upgrade Mechanism Unknown + +## What the Problem Is + +The exact technical mechanism for upgrading a Citrix-delivered application is not yet confirmed. The working assumption is that it involves: + + - Copying files to the Citrix server + - Handling any locked executable (the running app may lock its own `.exe`) + +## What Needs to Happen + + - Confirm with ESS or Citrix documentation how Citrix app upgrades work + - Determine if there's a Citrix-native method (e.g. App Layering, PVS updates) vs. a simple file swap + - Understand how to handle sessions that lock the executable + +# Issue 7 — Test/Prod on Same Server + +## What the Problem Is + +Some customers may have their TEST and PROD instances on the same physical server, differentiated only by install path or port. + +## Impact + +A misconfigured manifest or script bug could cause a PROD deployment when a TEST deployment was intended. + +## Mitigations + + - The manifest must clearly separate TEST and PROD config even if they share a server + - Consider an explicit confirmation prompt when deploying to PROD + - AzDO approval gates for PROD stages (not yet designed) + - The VALIDATE stage should catch obvious misconfigurations + +# Risk Register Summary + +| \# | Issue | Severity | Status | Mitigation | +| -- | -------------------------------- | -------- | ----------------- | ------------------------------------- | +| 1 | DB schema inconsistency | High | Unresolved | dacpacReady flag, manual DB fix first | +| 2 | Flat DLL artifact | Medium | Unresolved | Needs DLL-to-app mapping | +| 3 | No test environment | High | TBC | Treat all runs as live | +| 4 | No local execution | Medium | Inherent | Unit test Tier 3 with Pester | +| 5 | Cycle checks need UI | Medium | Manual short-term | Automate later; manual prompt now | +| 6 | Citrix upgrade mechanism unknown | Medium | TBC | Investigate with ESS | +| 7 | Test/Prod on same server | Medium | Design needed | Manifest separation, approval gates | diff --git a/Career/20260410122935-esp_open_questions.md b/Career/20260410122935-esp_open_questions.md new file mode 100644 index 0000000..9284e1b --- /dev/null +++ b/Career/20260410122935-esp_open_questions.md @@ -0,0 +1,171 @@ +# Overview + +These are items explicitly marked as TBC or not yet designed in the handover material. They represent real blockers or gaps that need to be resolved before the pipeline is complete. Use this file to track answers as they are determined. + +# Q1 — What Is the Rollback Plan? + +## The Question + +If a deployment fails partway through (e.g. the dacpac succeeds but a service won't start), how do we restore the system to its pre-deployment state? + +## Why It's Hard + + - The database cannot be easily rolled back without a restore from backup + - Files may have been partially swapped + - Citrix sessions may have been killed already + +## Things to Consider + + - Mandatory pre-deployment backup of database AND application files + - Snapshot-based restore if the server supports it (e.g. VM snapshots) + - A dedicated "rollback" stage that the pipeline can call + - Whether rollback is always manual or can be automated + +## Answer (Fill In When Known) + +Not yet determined. + +# Q2 — What Is the Test Plan? + +## The Question + +How will the pipeline and scripts themselves be tested before being used against customer environments? + +## Things to Consider + + - Unit tests for Tier 3 functions using Pester + - Integration tests against a test environment (once available — see Issue 3) + - Dry-run mode where the pipeline goes through all steps but prompts instead of acting + - Staged rollout: run full pipeline against ROMAC DEV first before any other environment + +## Answer (Fill In When Known) + +Not yet determined. + +# Q3 — What Is the Final Scope? + +## The Question + +Will the pipeline eventually cover: + + - All customer environments (not just ROMAC DEV)? + - Production environments? + - Environments hosted in ESS's own data centre (not Microlise's)? + +## Current Scope + +ROMAC DEV only, as an initial target. + +## Answer (Fill In When Known) + +Not yet determined — pending business decision. + +# Q4 — How Exactly Are Citrix Apps Deployed/Upgraded? + +## The Question + +What is the technical mechanism for upgrading a Citrix-delivered application? + +## Working Assumption + +A file copy to the Citrix server, with handling for the running executable being locked by active sessions. + +## Things to Investigate + + - Does Citrix use App Layering or Provisioning Services (PVS)? If so, the upgrade process is very different from a file copy + - Are there Citrix-native PowerShell cmdlets for managing published apps? + - How are locked executables handled — do sessions need to be fully terminated first, or is there a staging mechanism? + - Who at ESS has this knowledge? + +## Answer (Fill In When Known) + +Not yet determined. + +# Q5 — What Is the Manifest Structure? + +## The Question + +What does the manifest config file look like? What format, what fields, where is it stored? + +## See Also + +[ESP — The Manifest](id:9bf19a8b-5581-4be8-9892-913c51df0128) contains a proposed structure. This needs to be validated against actual deployment requirements and agreed by the team. + +## Answer (Fill In When Known) + +Proposed in [ESP — The Manifest](id:9bf19a8b-5581-4be8-9892-913c51df0128) - not yet confirmed. + +# Q6 — How Are Build Artifacts Mapped to Applications? + +## The Question + +Given that the ESP build produces a flat folder of DLLs, how do we determine which DLLs belong to which application? + +## Things to Investigate + + - Does the build process have any metadata or manifest about what it produced? + - Can the build pipeline be modified to output per-application folders? + - Is there existing documentation from ESS about which files belong where? + - Can we infer the mapping from the existing install directories on ROMAC DEV? + +## Answer (Fill In When Known) + +Not yet determined. + +# Q7 — Full Detail of Tier 2 & Tier 3 Functions Needed + +## The Question + +The handover defines the architecture but not the complete list of all Tier 2 logic and Tier 3 functions needed for a full deployment. What is the complete list? + +## Approach to Resolve + + - Work through each deployment stage manually with a human operator + - Document every step they perform + - Each manual step maps to at least one Tier 3 function + - Aggregate these into a complete function inventory + +## Answer (Fill In When Known) + +Ongoing — to be determined through walkthroughs with ESS/ops. + +# Q8 — What Are All the Prerequisites? + +## The Question + +The PREREQUISITES stage is described as checking "an extensive list" of prerequisites, but the full list has not been documented. + +## Things to Investigate + + - What software must be installed on each server type (app server, DB server, Citrix server)? + - What Windows configuration must be in place? + - What network connectivity is required? + - What service accounts / permissions must exist? + - What environment labels/flags are needed? + +## Answer (Fill In When Known) + +Not yet documented. + +# Q9 — How Should Secrets Be Managed? + +## The Question + +The manifest and scripts will need credentials (DB passwords, server credentials, service account passwords). Where do these live and how are they accessed securely? + +## Options + + - AzDO secret pipeline variables + - Azure Key Vault (referenced by name in manifest, retrieved at runtime) + - Windows Credential Manager on a self-hosted agent + - Encrypted secrets in the repo (not recommended) + +## Answer (Fill In When Known) + +Not yet determined. + +# Resolved Questions + +| \# | Question | Resolved Date | Answer | +| -- | -------------------------------------------------- | ------------- | ------ | +| | (none resolved yet) | | | diff --git a/Career/20260410123131-esp_glossary.md b/Career/20260410123131-esp_glossary.md new file mode 100644 index 0000000..76a952b --- /dev/null +++ b/Career/20260410123131-esp_glossary.md @@ -0,0 +1,197 @@ +# Overview + +Reference glossary for all technical and project-specific terms used across the ESP deployment pipeline project. Alphabetically ordered. + +# A + +## Agent (AzDO) + +The machine that executes an AzDO pipeline. Can be **Microsoft-hosted** (a fresh Azure VM for each run) or **self-hosted** (a persistent machine you manage). +For deploying to customer servers, a self-hosted agent is required to have network access to those servers. + +## Artifact (Build) + +The output of a build pipeline — typically compiled binaries, DLLs, config files, etc. packaged and stored so a deployment pipeline can consume them. +The ESP build artifact is currently a flat folder of DLLs. See [ESP — Known Issues & Risks](id:53d803aa-7e6f-44ed-8d99-a243a5ed5aab) + +## AzDO / Azure DevOps + +Microsoft's DevOps platform. Provides Repos (Git), Pipelines (CI/CD), Boards (work tracking), and Artifacts (package storage). The ESP pipeline lives here. + +# B + +## BACPAC + +A SQL Server package format that includes both schema **and** data. Useful for backup/restore. Contrast with Dacpac (schema only). + +## Build Pipeline + +An AzDO pipeline that compiles source code and produces a build artifact. The ESP build pipeline is `ESS.esp Main Build`. + +# C + +## CI/CD + +**Continuous Integration / Continuous Delivery (or Deployment)**. The practice of automatically building, testing, and deploying software whenever changes are made. AzDO pipelines implement this. + +## Citrix + +A technology platform that delivers desktop applications to users remotely. The application runs on a Citrix server; users see and interact with it via the Citrix Workspace client. Three ESP applications (VPlanner, VPlannerAdmin, ImportApp) are delivered this way. + +## Cycle Checks + +Post-deployment validation checks for ESP that involve navigating through the Citrix application UI to verify the system is functioning correctly. These are extensive and difficult to automate. See [ESP — Deployment Stages](id:cd3cd02f-c9b3-4455-8ce5-b2a5aa458fed). + +# D + +## Dacpac + +**Data-tier Application Package**. A `.dacpac` file that represents the desired schema of a SQL Server database. Deployed using `sqlpackage.exe`, which calculates the difference between desired and actual schema and applies it. See [ESP — Database & Dacpac](id:46add22d-e562-4e3a-a301-d4aea2552952). + +## `dacpacReady` + +A flag in the manifest (see [ESP — The Manifest](id:9bf19a8b-5581-4be8-9892-913c51df0128)) that indicates whether a customer's database has been manually aligned to be compatible with the dacpac. Must be `true` before the pipeline will attempt database deployment. + +## Deploy Pipeline + +The AzDO pipeline responsible for deploying ESP to customer environments. Currently: `ESS.esp Deploy`. This is the primary pipeline your team owns. + +## DLL + +**Dynamic Link Library**. A compiled Windows binary file (`.dll`) containing reusable code. Windows Services and other Windows applications are composed of DLLs. Deploying a new version means replacing old DLLs with new ones. + +# E + +## Environment + +In the context of ESP, an environment is a specific deployment target for a customer (e.g. DEV, TEST, PROD). A customer may have multiple environments. Each environment has its own manifest. + +## ESP + +The application suite built by ESS. Delivered to customers on a per-customer basis. Contains databases, Windows Services, and Citrix applications. See [ESP Applications](id:091e9bbc-0c9f-43da-81f9-464882c2a15b). + +## ESS + +The company that built ESP. Formerly had a larger engineering team; now reduced. Has a data centre of their own where some customer instances are hosted. + +## EspBroker + +The SQL Server database used by ESP. The only database in the suite currently. Deployed via dacpac. + +# G + +## Grace Period + +In the context of Citrix deployment: the time given to active Citrix users to save their work and quit before their session is forcibly terminated to allow the upgrade to proceed. + +# J + +## Job (AzDO) + +A unit of work within an AzDO pipeline stage. Each job runs on a single agent. A stage can have multiple parallel jobs. + +# L + +## Layer / Tier + +The architectural layers in the PowerShell script design. Tier 1 = entry point; Tier 2 = orchestration; Tier 3 = isolated worker functions. See [ESP — PowerShell Script Architecture](id:bf63b6c5-f32f-462e-82ad-8d4a15f7ba48). + +# M + +## Manifest + +A configuration file defining a specific customer environment. Used by the deployment scripts to know what to deploy, where, and how. +See [ESP — The Manifest](id:9bf19a8b-5581-4be8-9892-913c51df0128). + +## Microlise + +The company the team works for. The ESP pipeline project is being done by Microlise engineers to assist ESS. + +## Module (PowerShell) + +A packaged collection of PowerShell functions, stored in a `.psm1` file. Modules are loaded with `Import-Module`. The Tier 2 and Tier 3 scripts are structured as modules. + +# P + +## Pester + +The standard PowerShell testing framework. Used for writing unit tests for PowerShell functions. Particularly useful for testing Tier 3 functions in isolation. + +## Pipeline (AzDO) + +An automated workflow defined in YAML that can build, test, and deploy software. Triggered by events (code pushes, schedules, manual runs). + +## PowerShell + +Microsoft's scripting language and shell, built on .NET. Used for all deployment scripting in this project. + +## Prerequisites + +The PREREQUISITES deployment stage — checks that the target environment has all required software, configuration, and connectivity before deployment begins. + +## `.psm1` + +The file extension for a PowerShell module file. + +# R + +## Remote Session (PowerShell) + +A PowerShell remoting session to another machine, created with `New-PSSession`. Allows you to run PowerShell commands on a remote server. Required for managing Windows Services and files on customer servers. + +``` powershell +$session = New-PSSession -ComputerName "ROMAC-DEV-APP01" +Invoke-Command -Session $session -ScriptBlock { Get-Service } +``` + +## ROMAC DEV + +The initial target customer environment for the pipeline. ROMAC is the customer name; DEV is the environment tier. + +# S + +## Schema (Database) + +The structure of a SQL Server database — its tables, columns, indexes, constraints, stored procedures, etc. The dacpac encodes the desired schema. + +## Self-Hosted Agent + +An AzDO agent running on a machine you manage (rather than a Microsoft-managed Azure VM). Required for this project to have network access to customer servers. + +## `sqlpackage.exe` + +Microsoft's command-line tool for deploying dacpac files to SQL Server. + +## Stage (Deployment) + +One of the five phases of the ESP deployment: VALIDATE, PREREQUISITES, PREDEPLOY, DEPLOY, POSTDEPLOY. Each can be run independently. See [ESP — Deployment Stages](id:cd3cd02f-c9b3-4455-8ce5-b2a5aa458fed). + +## Stage (AzDO Pipeline) + +A high-level grouping within an AzDO YAML pipeline, containing jobs. Not the same as a deployment stage — naming coincidence. + +# T + +## Team Ludo + +The Microlise team that preceded your team on this project. They built the ESP suite in AzDO, started deployment work, and built the dacpac. Now reassigned to paid customer work. + +## Tier 1 / 2 / 3 + +See **Layer / Tier** above and [ESP — PowerShell Script Architecture](id:bf63b6c5-f32f-462e-82ad-8d4a15f7ba48). + +## TMC + +Another Microlise product, referenced in the handover as a comparison point for how ESP works (per-customer deployment, multiple environments, etc.). + +# W + +## Windows Service + +A background process that runs on a Windows server, managed by the Windows Service Control Manager. Can be started, stopped, and queried via PowerShell. Six ESP services exist: DLService, EmailService, ExPlService, MISService, OutboundService, ULService. + +# Y + +## YAML + +**YAML Ain't Markup Language**. A human-readable data format used for AzDO pipeline definitions. Stored in the repo as a `.yml` file. diff --git a/Career/20260413120825-windows_services.md b/Career/20260413120825-windows_services.md new file mode 100644 index 0000000..6144288 --- /dev/null +++ b/Career/20260413120825-windows_services.md @@ -0,0 +1,75 @@ +# Summary + +Windows Services are essential components of the Windows operating system that run in the background to perform critical system functions. They operate without user interaction, starting automatically during system boot and continuing until shutdown. Managed by the Service Control Manager, they handle tasks such as network connectivity, hardware management, and system security. Administrators can manage services using tools like Services.msc, command-line utilities, and PowerShell for automation. + +# Definition and Purpose + +****Windows Services**** are long-running executable applications that operate in the background of the Windows operating system. They are designed to perform specific system-level functions without requiring user interaction, making them essential for the stable and continuous operation of Windows. + +These services typically start during system boot and continue running until the system shuts down. They handle core tasks such as ****network connectivity****, ****hardware management****, ****system security****, and ****scheduled operations****. + +## Key Characteristics + +Windows Services differ from regular desktop applications in several important ways: + + - ****No User Interface (UI)****: They run without a graphical interface, operating silently in the background. + - ****Automatic Startup****: Can be configured to start automatically when the system boots. + - ****Session Independence****: Run in their own Windows sessions (often Session 0), independent of logged-in users. + - ****Service Control Manager (SCM)****: Managed by the SCM (`services.exe`), which handles starting, stopping, and monitoring services. + - ****Security Context****: Operate under specific user accounts such as ****SYSTEM****, ****Local Service****, or ****Network Service****, allowing them to function even when no user is logged in. + +Prior to Windows Vista, some services could interact with the desktop, but this capability has been largely disabled due to ****Windows Service Hardening**** and ****Session 0 Isolation**** for security reasons. + +## Common Examples + +Many critical Windows functions are implemented as services. Some well-known examples include: + + - ****Print Spooler****: Manages print jobs and printer communication. + - ****DHCP Client****: Obtains IP addresses dynamically for network connectivity. + - ****Windows Update****: Downloads and installs system updates. + - ****Task Scheduler****: Executes tasks at predefined times or events. + - ****DNS Client****: Resolves domain names to IP addresses. + - ****Windows Time****: Synchronizes the system clock with internet time servers. + +Third-party applications like antivirus software, database servers, and cloud sync tools also often install their own services to ensure continuous background operation. + +## Management Tools + +Windows provides several built-in tools to manage services: + + - ****Services.msc****: The graphical ****Services Manager**** accessible via Run command or Control Panel. It allows viewing, starting, stopping, and configuring services. + + - ****Command Line (sc.exe)****: A powerful tool for querying and controlling services. For example: + + ``` cmd + sc query eventlog + sc start "Print Spooler" + ``` + + - ****PowerShell****: Offers cmdlets like `Get-Service`, `Start-Service`, `Stop-Service`, and `Set-Service` for automation and scripting. + + - ****Task Manager****: In Windows Vista and later, it can display and control services under the "Services" tab. + + - ****MSConfig****: Allows enabling or disabling services at startup for troubleshooting. + +## Startup Types + +Each service has a ****startup type**** that determines when and how it starts: + + - ****Automatic****: Starts immediately during system boot. + - ****Automatic (Delayed Start)****: Starts shortly after boot to improve startup performance. + - ****Manual****: Starts only when triggered by a user, application, or system event. + - ****Disabled****: Cannot be started under any circumstances until re-enabled. + +These settings help balance system performance and functionality, especially during boot. + +## Administration Methods + +Administrators can manage services using various interfaces: + + - ****Graphical****: Services snap-in (`services.msc`) provides full configuration including recovery actions, logon accounts, and dependencies. + - ****Command-Line****: `sc.exe` allows scriptable service management and remote administration. + - ****PowerShell****: Ideal for automation and bulk operations across multiple systems. + - ****Remote Management****: The Services MMC snap-in can connect to remote computers on the network. + +Services can also be created programmatically using the ****System.ServiceProcess**** namespace in .NET, or wrapped using tools like ****SrvAny.exe**** from the Windows Resource Kit. diff --git a/Career/20260413154901-dll.md b/Career/20260413154901-dll.md new file mode 100644 index 0000000..84fe0ac --- /dev/null +++ b/Career/20260413154901-dll.md @@ -0,0 +1 @@ +A Dynamic Link Library (DLL) is a type of file that contains code and data that can be used by multiple programs simultaneously. It allows developers to modularise their applications, enabling code reuse and efficient memory usage. DLLs are commonly used in Windows operating systems, but they can also be found in other platforms. diff --git a/Career/20260505121135-seb_search_improvements.md b/Career/20260505121135-seb_search_improvements.md new file mode 100644 index 0000000..1c55beb --- /dev/null +++ b/Career/20260505121135-seb_search_improvements.md @@ -0,0 +1,180 @@ +# Overview + +DBAs are seeing high CPU and memory usage on SQL production servers due to SEB +Search. Free text search uses the `LIKE` operator against RouteID, JourneyAlias, +formatted driver name, consignment references, site names etc. The `LIKE` operator +scans table indexes and is expensive. + +By adding a Couchbase caching layer, fewer requests reach SQL Server. + +**Scope:** FreeText Search redirection only — NOT Saved Search. +FreeText accounts for **85–90%** of all SEB search requests. + +## What is Free Text vs Saved Search? + + - **Free Text Search** — user types in the Search box AND selects "All Journeys" + - **Saved Search** — user selects any option other than "All Journeys" from the + "Search by preset" dropdown + +# Design + +Journey details are pushed to Couchbase when journeys are created or modified +in the TMC database, via the Outbox Processor Service → Kafka → Couchbase. + +When a WebPortal user searches, the request goes to the multi-tenanted +`SEBSearchWebApi` (deployed on OpenShift), which returns matching Journey IDs. +Those IDs are passed back to the TMC Database for visibility/config filtering, +then journey data is retrieved and displayed. + +# Release + + - **Available from:** 25.8 + - **Feature Flag:** `SEB Search Improvement` + +## Feature Flag Behaviour + + - Automatically enabled for existing customers by the Outbox Processor Service + after existing journey cache is built (CERT, UAT, LIVE). + - For **new customers** using SEB: flag must be enabled **before** creating any + journeys. + +# Rollout Instructions (TechOps) + +## Pre-requisites + +1. TMC system upgraded to **25.8** +2. `SSO.Alchemy.ApiKey` updated with the IDAM API Key +3. `SEBSearchWebApi.ApiUrl` updated with the correct SebSearchWebAPI URL + +## Steps + +The Outbox Processor Windows service is **disabled by default**. +Enable it for the customer, preferably between **1:00 AM – 4:00 AM**. + +## SEBSearchWebAPI URLs + +| Environment | URL | +| ----------- | --------------------------------------------------------- | +| DEV | | +| CERT | | +| UAT | | +| PROD | | + +# UAT Refresh Instructions (DBA) + +Each time a TMC database is restored from PROD to UAT, run the following as +part of the UAT refresh process. **Do NOT use TRUNCATE.** + +``` sql +DELETE FROM dataSync.tbl_SebSearchCachePublishedJourneys +``` + +This table records journeys cached to Couchbase. Deleting it causes the Outbox +Processor to rebuild the cache for the UAT environment. + +# New Components + +## Outbox Processor Service + + - Reads from `dataSync.tbl_OutboxEvent` in the TMC database + - Sends Journey metadata to a Kafka topic + - Polls every **30 minutes** (default) for unpublished journeys + - Deployed on: TMC Application Server + +## Kafka Connector (Sink) + + - Reads from Kafka topic, writes to Couchbase + - Multi-tenant + - Pipeline name: `env_niot_journey_sebsearch_couchbase` + - Very fast — consumed 1M messages in 20–25 minutes in tests + +## SEBSearchWebApi + + - Serves journey search requests from WebPortal + - Returns list of Journey IDs + - Secured via IDAM API keys + - Deployed on OpenShift as multi-tenant service + - Capable of automatic horizontal scaling + +# Journey Metadata Cached + +The following fields are cached (same as existing FreeText search queries): + + tbl_Journeys.RouteID + tbl_Journeys.JourneyAlias + fn_GetConcatResourceNameWithFormattedDriver + └─ tbl_Drivers.DisplayName / FullName (depending on DriverNameFormat config) + fn_GetConcatConsignmentReferences + └─ tbl_ConsignmentHeader.OrderRef + └─ tbl_ConsignmentHeader.CustomerOrderRef + └─ tbl_ConsignmentHeader.DocumentID + └─ tbl_ConsignmentHeader.ConHeaderInfo1–5 + tbl_JourneyDrops.DropPointID + tbl_DropPoints.DropName + tbl_SiteType.SiteTypeName + tbl_Journeys.StartTime + tbl_Journeys.EndTime + +# Performance Data + +## Journey Counts (as of 17 June 2025) + + - Total journeys across estate (up to archiving period): **8.7 million** + - Top 5 customers by journey count (descending): + 1. TMCEUROCARPARTSSTORESL + 2. TMCASDAGHS + 3. TMCALLIANCEHEALTHCAREL + 4. TMCGSFCARPARTSL + 5. TMCTESCOUKL + - ASDA and Tesco UK have shorter retention periods than the other top-5 + +## SEB Search Hit Analysis (as of 18 June 2025) + + - **5.5 million** SEB search hits in the prior month (excl. SEB Refresh hits) + - Weekdays \> weekends; **Thursday** is peak day + - Peak working hours: **6 AM – 6 PM** + - Peak load (9 AM – 2 PM): \~15,000 searches/hour across the estate + - ECPS is the biggest contributor during peak hours + - Across the full day: Sainsbury's highest, then Tesco UK + - Max observed: **317 search requests in a single minute** (Thursday) + +## Test Results + +| Component | Result | +| -------------------- | ---------------------------------------------------------- | +| Outbox Processor | 1M journeys pushed in 3–4 hours | +| Kafka Sink Connector | 1M messages consumed in 20–25 minutes (even after backlog) | +| SEBSearchWebApi | 1,863 req/min capacity vs 438 req/min current load | + +# Component Stack (all must be running) + +1. OutboxProcessorService +2. TMC Database +3. Couchbase +4. Kafka +5. Sink Connector +6. IDAM API Key Authorisation Service (AKAS) +7. OpenShift cluster +8. SEBSearchWebApi +9. TMC WebPortal +10. Journey groups / user visibility config + +# Known Existing SEB Behaviours (pre-feature) + + - Clicking the **first** number in "Loaded X of Y journeys" only updates that + number — the search result does not change. + - Clicking the **second** number (Y) loads all Y journeys. + - SEB auto-refresh only refreshes displayed data — new/removed journeys + matching the same criteria are not added/removed dynamically. + +# Dashboards & Pipelines + + - **SEBSearchWebApi dashboard** — available in monitoring tooling + - **Sink connector pipeline:** `env_niot_journey_sebsearch_couchbase` + +# [Links (internal)](https://microliseuk.sharepoint.com/sites/JourneyManagement/SitePages/SEB-Search-Cache.aspx?ga=1) + + - Local dev setup (Kafka, connector, Couchbase): \[see internal wiki\] + - Local dev setup (OutboxProcessorService): \[see internal wiki\] + - Enable SebSearchCaching on new TMC systems: \[see internal wiki\] + - Troubleshooting guide: \[see internal wiki\] diff --git a/Career/20260512121806-data_camp_ai_training.md b/Career/20260512121806-data_camp_ai_training.md new file mode 100644 index 0000000..2d8998c --- /dev/null +++ b/Career/20260512121806-data_camp_ai_training.md @@ -0,0 +1,161 @@ +**TITLE: Introduction to AI for Work** + +# Summary + +## Learning Machines + +****AI Fundamentals**** + + - Artificial intelligence enables computer systems to perform tasks typically associated with human intelligence, such as learning, reasoning, and decision-making. + - AI has actually been around for many years and is already integrated into your daily life in ways you might not have realized. + +****Traditional Programming**** + + - Before modern AI, engineers created intelligent systems by explicitly programming step-by-step procedures. + - The challenge is that for many important tasks, we simply can't spell out the procedure—a human expert may excel but can't articulate the instructions for a computer. + +****Machine Learning**** + + - Instead of programming step-by-step procedures, machine learning enables computers to learn from examples. + - AI fundamentally works through pattern recognition.. + - During Training: The system recognizes and learns patterns from examples. + - During Operation: The trained system receives new cases and compares them against the patterns it learned during training to make decisions. + +## Generative AI + +****The Generative AI Breakthrough**** + + - While AI researchers have been making steady progress for decades, late 2022 was the tipping point when systems like ChatGPT became good enough for public release. + - Generative AI systems generate new content rather than simply analyzing existing information. + +****Large Language Models**** + + - Large Language Models (LLMs) are the most important type of generative AI for your work—the technology powering tools like ChatGPT, Claude, and Gemini. + - They can communicate fluently in natural language. More importantly they exhibit common sense and logical reasoning capabilities. + +****How LLMs Work**** + + - Large Language Models (LLMs) work like other machine learning systems. They learn patterns from data during training. + - What makes them special is scale. They’re trained on huge amounts of text from the internet, books, articles, and more. + - Because of this broad training, they can handle many different kinds of tasks — writing, summarizing, coding, explaining, etc. + - When you give them a prompt or question, they use learned patterns to generate a fitting response. + +****Beyond Language Models**** + + - Generative AI also includes systems that create and modify images, as well as video. + - Image generation is a key capability for many professionals, including marketing, product design, and documentation. + - Video generation is a new frontier in AI, but it's already being used for training materials and marketing videos. + +## The Opportunity + +****Understanding AI at Work**** + + - Today's AI systems excel at doing tasks, not taking over entire jobs. + - AI doesn't boost productivity on all tasks—only those within AI's capability boundaries where it provides significant value. + - Even for tasks where AI helps, producing useful output requires human oversight and judgment. + +****The Real Opportunity**** + + - AI is raising the bar for what any professional should be able to accomplish. + - The real question isn't "Will AI take my job?" but "Will I be one of the people who can work effectively with AI?" + +****Significant Benefits**** + + - When used effectively within its capability boundaries, AI helps you accomplish significantly more, at higher quality, while making work more engaging. + - Most professionals aren't leveraging AI well yet. By being here, you're setting yourself up to be ahead. + +## How AI Can Help You + +****Execution**** + + - AI excels at executing knowledge work tasks where you know what needs to be done and the work is at a level you'd delegate to a capable junior teammate + - In practice: You provide clear instructions, AI carries it out, you review the output + - This dramatically reduces time on routine tasks and frees you for higher-value work + +****Thought Partnership**** + + - AI serves as an exceptional brainstorming partner when you don't know what needs to be done—like turning to a creative colleague to think through tough problems + - Particularly effective for diagnosing unclear problems, exploring solution possibilities, and weighing difficult decisions + - Make it a habit to bring AI to the table when facing complex challenges + +****Refinement**** + + - AI helps you improve your work by providing high-quality, objective feedback—pointing out weaknesses and suggesting concrete improvements + - In practice: Share your work, specify what feedback you need, and AI provides specific suggestions + - Make it a habit to seek feedback from AI on your work + +****Continuous Learning**** + + - AI can explain any concept clearly and immediately, adapting its approach until it clicks for you + - AI is an effective teacher: adjusts to your level, welcomes all questions without judgment, and is always available + - Note: AI cannot replace structured learning programs that require expert-designed progressions and hands-on practice + +## Working with AI Effectively + +****Core Collaboration Principles**** + + - Think of working with AI as collaborating with a colleague rather than using a tool. This collaboration mindset underlies everything about working effectively with AI. + - Communicate Effectively: + - Use clear, unambiguous language—no special phrases or magic words are required. + - Give AI sufficient detail to accomplish the task successfully. + - Iterate: AI's first output is rarely perfect. Continue the conversation and guide AI toward what you need through multiple rounds of feedback. + +****Communication Framework**** + + - The Ask: What exactly do you want the AI to do? Be specific and clear about the task or outcome. + - The Requirements: What does the output need to satisfy? Include focus, boundaries, format, style, and other specific needs. + - The Context: What does AI need to know about your specific situation? Why you need this, how it will be used, and relevant background details. + - The Examples (optional): What does success look like? Show the AI what you want through concrete examples—especially valuable for visual/structural requirements ( format, layout) and qualitative/subjective requirements (style, tone, quality standards). + +****The Practical Test**** + + - If you walked your request to a competent junior teammate, can they complete the task with the information you provided? If not, AI probably can't either. + +****Practical Tactics**** + + - Content over polish: Focus on including the right information (ask, requirements, context) rather than perfecting the writing. AI handles spelling mistakes, broken sentences, and disorganized thoughts—what matters is having the key details, not perfect prose. + - Start simple and build: Begin with a basic request and add more detail based on what you get back. Don't stress about getting everything right upfront. + - Ask AI what it needs: If unsure what information to provide, ask directly: "What would you need to know to help me with this?" + - Use different modalities: Dictate your requests, provide screenshots or photos, or mix text, voice, and images as needed. + +****Practice**** + + - The best way to get better at using AI is to use AI. + - Start with low-stakes tasks where you can experiment without pressure. + +## Working with AI Responsibly + +****AI's Limitations**** + + - Knowledge fabrication: AI can confidently produce false information that sounds completely plausible + - Recency ignorance: AI works with outdated knowledge from its training period + - Biased outputs: AI can unfairly favor or underrepresent certain groups, viewpoints, or aesthetics + - Sycophantic outputs: AI tends to tell you what it thinks you want to hear + +****Why These Happen**** + + - These limitations result from AI's training data, training process, and pattern-matching approach + - AI labs are actively improving these issues with each generation, but they haven't been eliminated + - Your judgment and oversight remain irreplaceable safeguards when using AI + +****Review AI Outputs**** + + - Maintain critical assessment and healthy skepticism with AI outputs + - Verify factual claims (dates, statistics, citations, technical details, recent information) + - Ask AI to search the web and cite sources, especially for factual or recent information + - Watch for bias: Are perspectives missing? Would this be fair to all affected groups? + - Watch for sycophancy: Is AI telling me what I want to hear? + - Calibrate scrutiny to stakes (more rigorous for high-stakes decisions) + +****Seek Critical Perspectives**** + + - Counter bias: Ask AI to consider different viewpoints ("What perspectives might be missing?" "How might this affect different groups differently?") + - Counter sycophancy: Explicitly request critical feedback ("What are the weaknesses?" " What could go wrong?" "What assumptions might be wrong?") + - For important decisions, consult people from diverse backgrounds who can offer genuine pushback + +****Privacy Risks**** + + - Conversations may become training data; databases can be breached + - Follow organizational policies and avoid sharing sensitive information + - You can usually achieve your goal without exposing private details—use generic examples or anonymized data instead + - Use privacy protections (enterprise versions, private modes, opt-out settings) diff --git a/Career/20260513115140-powershell_moc.md b/Career/20260513115140-powershell_moc.md new file mode 100644 index 0000000..031d469 --- /dev/null +++ b/Career/20260513115140-powershell_moc.md @@ -0,0 +1,8 @@ +# Articles/Resources: + + - [On Synopsis and help syntax](https://gist.github.com/MyITGuy/18ea0f54d2accc0eb8e7ae1521952de0) + - [On backticks and why they are bad](https://get-powershellblog.blogspot.com/2017/07/bye-bye-backtick-natural-line.html#naturallinecont) + +# Random: + + - [Reddit post on backticks](https://www.reddit.com/r/PowerShell/comments/6q4q0f/bye_bye_backtick_natural_line_continuations_in/) diff --git a/Career/20260513142352-aritificial_intelligence.md b/Career/20260513142352-aritificial_intelligence.md new file mode 100644 index 0000000..f07c210 --- /dev/null +++ b/Career/20260513142352-aritificial_intelligence.md @@ -0,0 +1,15 @@ +# Codex Prompt plan \<2026-05-13 Wed\> to \<2026-05-20 Wed\> + +## DONE \[\#A\] Remove/fix the territory anchors + +## DONE \[\#A\] Clean up the hidden authoring service, and remove relationships / threads, but introduce a story like connection + +## DONE \[\#B\] Move the authoring service into its own repo + +## DONE \[\#C\]: Clean up the code (js, css files) + +## DONE \[\#A\]: Improve the search UI in org web + +## DONE \[\#A\]: Improve the search UI in org roam + +## DONE \[\#B\]: Improve design of org roam diff --git a/Career/20260527152602-microlise_xss_rom_pentest.md b/Career/20260527152602-microlise_xss_rom_pentest.md new file mode 100644 index 0000000..34d832d --- /dev/null +++ b/Career/20260527152602-microlise_xss_rom_pentest.md @@ -0,0 +1,429 @@ +# Links: + + - [Cross Site Scripting (XSS)](id:01c89142-7e14-42b2-bd01-743656908fd2) + +# Metadata + + - Name + Understanding Session XSS + - Overview + Pages 33–34 document an informational “Session stored XSS” finding on the TMC Schedule Execution Board. The issue is real (unescaped user input in a JavaScript context) but impact is limited because only the submitting user’s session is affected—classic self-XSS, not cross-user attack. + +## Todos + + - \[X\] review-finding + Read pages 33–34 and map finding to SaveSearchCriteriaToSession + ScheduleExecutionBoard.aspx flow + - \[X\] locate-source + Open TMC Web Portal repo and find session save + inline script render for date/orderID/time + - \[ \] remediate-encode + Apply HttpUtility.JavaScriptStringEncode to all session values in SetupControls() (\~1849-1879) + - \[ \] remediate-validate + Add server-side validation in SaveSearchCriteriaToSession before writing SEBSessionState + - \[ \] remediate-retest + Retest with direct POST payload + normal UI search flow on ScheduleExecutionBoard + +# Understanding the Session Stored XSS Finding (Pages 33–34) + +## Where this sits in the report + +The [Microlise TMC PO WA April 2026 v1.0.pdf](d:/_dev/_misc/Pentest-04-26/Microlise%20TMC%20PO%20WA%20April%202026%20v1.0.pdf) lists **14 findings** total. Pages 33–34 ([33-34.pdf](d:/_dev/_misc/Pentest-04-26/33-34.pdf)) are the last technical finding before “END OF DOCUMENT”: + +| Field | Value | +| ----------- | ------------------------------------------------------------------------------------------- | +| Title | **Session stored XSS** | +| Severity | **Informational** (lowest tier; 4 informational findings in the report) | +| Status | Open | +| CWE | [CWE-79](https://cwe.mitre.org/data/definitions/79.html) — Improper Neutralization of Input | +| Environment | `cert.microlise.com` (cert/UAT), path prefix `/PENTEST/TMCWebPortal/` | + +Higher-severity items in the same report (SQLi, IDOR, BFLA, etc.) are separate; this finding is documented as **technically valid but low business risk**. + +----- + +## What XSS is (general) + +**Cross-Site Scripting (XSS)** means untrusted data ends up in a web page in a way the **browser treats as executable JavaScript**, instead of inert text. + +The name “cross-site” is historical: classic attacks trick a **victim** into loading a page on **your** app so script runs in **your** origin (stealing session cookies, performing actions as the user, etc.). + +Common types: + +| Type | Persistence | Typical delivery | +| ------------- | ------------------------------------- | -------------------------------- | +| **Reflected** | Not stored; one-off response | Malicious link/query param | +| **Stored** | Saved server-side (DB, file, session) | Victim loads a normal page later | +| **DOM-based** | Client-side only | Unsafe innerHTML, eval, etc. | + +**Defense in depth:** validate input on the server (whitelist formats), and **encode output** for the exact context (HTML, attribute, JavaScript string, URL). + +----- + +## What happened in *this* finding (TMC context) + +### Affected surface (source located) + +| Role | Path | +| --------------------------- | ---------------------------------------------------------------------------------------------------------- | +| Page + inline JS | [ScheduleExecutionBoard.aspx](d:/_dev/WebPortal/src/code/AmberWebUI/SEB/ScheduleExecutionBoard.aspx) | +| WebMethod + page properties | [ScheduleExecutionBoard.aspx.cs](d:/_dev/WebPortal/src/code/AmberWebUI/SEB/ScheduleExecutionBoard.aspx.cs) | +| Session storage | [SEBSessionState.cs](d:/_dev/WebPortal/src/code/AmberWebUI/SEB/SEBSessionState.cs) | + + - **Endpoint:** ASP.NET `[WebMethod]` `SaveSearchCriteriaToSession` on `ScheduleExecutionBoard.aspx` + - **Parameters:** JSON fields `date`, `orderID`, `time` (also `searchID`, `hours`, `quickSearch` in the same flow) + - **Host (pentest):** `cert.microlise.com`, path `/PENTEST/TMCWebPortal/SEB/...` + +### Attack flow (as tested) + +``` mermaid +flowchart LR + subgraph submit [Step1_Submit] + A[Tester sends POST directly] + B[SaveSearchCriteriaToSession] + C[Values stored in server session] + end + subgraph render [Step2_Render] + D[User loads ScheduleExecutionBoard.aspx] + E[Server embeds session values in script block] + F[Browser executes unescaped JS] + end + A --> B --> C + C --> D --> E --> F +``` + +1. **Save:** User (or tester) POSTs JSON to `SaveSearchCriteriaToSession`. The app saves search criteria into the **server-side session**. +2. **Render:** On the next load of `ScheduleExecutionBoard.aspx`, those values are written into the HTML **inside a `