[{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5kOM3LeUL0ahgdCGnSOycI","type":"Entry","createdAt":"2023-05-16T22:47:04.629Z","updatedAt":"2023-05-16T22:47:04.629Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Proactive safeguarding with Conduktor Gateway","description":"In this article, we discuss how safeguarding can be used to prevent outages and respond gracefully to change.","noIndex":false}},"slug":"proactive-safeguarding-with-conduktor-gateway","title":"Proactive safeguarding with Conduktor Gateway","navbarDescription":"Proactive safeguarding with Conduktor Gateway","description":"In this article, we discuss how safeguarding can be used to prevent outages and respond gracefully to change. Most organizations have some safeguarding policies in place, we've laid out the three most common stages of safeguarding and how Conduktor Gateway can help you prevent avoidable Kafka issues.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3rf2y35CciZLxgFKxTbK8H","type":"Asset","createdAt":"2023-05-16T14:26:17.509Z","updatedAt":"2023-05-16T14:26:17.509Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Safeguard blog hero","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3rf2y35CciZLxgFKxTbK8H/d354e8635d7a66f95a35d0fd2ff08b49/Safeguard.png","details":{"size":225504,"image":{"width":4096,"height":2160}},"fileName":"Safeguard.png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We recently launched ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/enhance-apache-kafka-with-conduktor-gateway/"},"content":[{"nodeType":"text","value":"Conduktor Gateway","marks":[],"data":{}}]},{"nodeType":"text","value":" to help you enhance Kafka functionality. You can now try our open-source Gateway on Conduktor ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/"},"content":[{"nodeType":"text","value":"Marketplace","marks":[],"data":{}}]},{"nodeType":"text","value":"! One of the ways that Gateway can enhance Kafka functionality is by enforcing Kafka best practices to safeguard against potential Kafka issues.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Safeguarding is at the intersection of data governance and enforcing technical best practices. Ensuring not only that the data in your system is healthy, valid, and useful but also that your ability to serve that data is not compromised by bad actors in the system. Effective safeguarding can help organizations:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Minimize outages","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Respond gracefully to change","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Reduce friction between domain teams","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is safeguarding?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Safeguarding, as the name suggests, is all about protection. Protection for the data that flows through your cluster and protection for the cluster itself. We like to define safeguarding as the practice of protecting your data and infrastructure from intentional and unintentional threats, both internal and external.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We've worked with many of our customers looking to improve their safeguarding best practices and identified three stages in their journey:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Safeguarding by habit","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Reactive automation","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Proactive automation","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In this article, we will look at these stages in detail, identify common themes and discuss ways in which to improve and progress to the next stage.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Safeguarding by habit","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Today, the majority of safeguarding policies look like this: a loose set of contracts enforced by humans. These often boil down to some documented practices that are periodically discussed and partially applied.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Safeguarding of this type often suffers from issues like:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Relevance - How well does the safeguarding rule fit the use case? Safeguarding by habit practice tends to be \"one size fits all\" and is not tailored or relevant for a particular technology/domain.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Timeliness - When in the development cycle is safeguarding being applied? Safeguarding by habit is often applied right at the end of the cycle when changes are difficult and costly to make.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Perception - Mostly due to the two factors above, these checks are seen as a necessary evil. \"Do we have to?\" and \"What can we get away with?\" are common phrases associated with safeguarding by habit actions.","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A perfect example of this is the \"production readiness checklist\". These are typically applied right at the end of a development cycle. Just before a new application is released to production you develop, test, and a week before you go live, you are presented with several boxes to tick. These generally suffer from all the issues detailed above and as such carry a bad reputation.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Reactive automation","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The obvious technology solution to any human-based process is to automate. Reactive automation takes the safeguarding by habit process discussed above and attempts to represent them in metrics that can be collected from your Kafka cluster. Corresponding thresholds and alerts are created on these metrics, which are triggered when they are breached. However, on detection of something irregular, the question becomes \"what's next?\"","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let's say there is a misbehaving client that is impacting the health of a Kafka cluster and, by association, the health of other applications connected to the cluster. Straightforward logic says you should turn the misbehaving application off. But what if it has dependencies? What if it is more mission-critical to the business than all of the applications it is impacting?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"These are complex decisions that need to be made on a case-by-case basis and need to be made quickly, as they are generally caught in production and have a wide impact radius. Even worse, the pressure associated with dealing with such incidents often leads to workarounds and temporary fixes that are never removed and end up dictating technical constraints for years to come.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"All of this combined adds up to tech debt, a lot of it.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Proactive automation","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Proactive is the gold standard of safeguarding but rarely seen in organizations. As with reactive automation, rules are created to detect irregular actions but, instead of being reported and analyzed in the future, these rules are enforced at the point of interaction with Kafka. Requests that do not meet the standards imposed by safeguarding rules can be rejected or even automatically corrected in a way as to make them compatible before reaching Kafka.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A good example of this is a topic creation request for say 50,000 partitions. This is very unlikely to be a good idea as it can be very resource intensive for the Kafka cluster and so, instead of rejecting the request the proactive automation could modify it to hit a maximum of 50 partitions. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When you contrast this to the reactive approach, you see the value of proactive automation. Reactive automation would have found the highly partitioned topic at some point in the future and determined it was against best practice. However, at this point, the topic holds data and the task of reducing partitions is difficult and time-consuming.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"How can you achieve proactive automation?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Now that we have established the reasoning behind proactive automation, let's look at how it can be achieved. This question soon becomes less of a \"how\" and more of a \"where\" it can be applied:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Within the Kafka Cluster (as a broker plugin)","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Within the applications (as a client plugin)","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Between Cluster and Client (as a proxy)","marks":[],"data":{}}]}]}]},{"nodeType":"heading-4","data":{},"content":[{"nodeType":"text","value":"Proactive automation within the Kafka cluster","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Deploying proactive automation as a broker plugin carries the advantage of being close to the data and also close to the running conditions of the cluster but has downsides in that the plugin operation is closely coupled to the broker operation causing issues in versioning and maintenance. Given the current state of Kafka, it is likely that the new plugin versions will incur Kafka broker restarts, etc., and all of the issues that can come along with this.","marks":[],"data":{}}]},{"nodeType":"heading-4","data":{},"content":[{"nodeType":"text","value":"Proactive automation within the application","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"On the opposite side of the scale is a client plugin that runs as close to the source of events as possible. This integrates well but creates a huge surface area to manage and maintain. Common strategies would expect at least one plugin per client and require that the plugin will support a range of Kafka clients (not just Java). Worse, as the server is not aware of the plugin it cannot ensure that clients are using it. Bad actors that wish to circumvent plugin restrictions can simply not implement it and go directly to the Kafka cluster.","marks":[],"data":{}}]},{"nodeType":"heading-4","data":{},"content":[{"nodeType":"text","value":"Proactive automation between the cluster and application","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A good balance between these two options is a Kafka proxy that sits between the client and the broker. This avoids a lot of the maintenance issues described (a proxy can be configured and restarted independently of the Kafka brokers) while still providing the functionality required. The proxy represents itself as a Kafka cluster to clients so can serve all traffic from any client type and, once a proxy is in place, direct access to the Kafka cluster can be disabled via the usual Kafka security mechanisms. ","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Introducing Conduktor Gateway","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Architecturally a proxy is the front-running solution but proxies can be very difficult to develop and the Kafka wire protocol is tricky to implement. To counter this, we introduced ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/platform/gateway/installation/opensource-install/"},"content":[{"nodeType":"text","value":"Conduktor Gateway","marks":[],"data":{}}]},{"nodeType":"text","value":", a Kafka proxy with the ability to enforce safeguarding rules. At a fundamental level, Gateway is the transport layer between your Kafka client applications and your Kafka clusters. This layer is enhanced with Conduktor Interceptors or plugins that you can install on Gateway. One such Interceptor is Safeguard, which can transform, project, or filter Kafka requests or responses with a set of rules before interaction with Kafka.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To take the example from before, to limit the number of partitions that can be created, Conduktor Gateway would execute an Interceptor on the CreateTopics request that would conditionally alter the partition count for the topic submitted in the request (from client to Gateway the request would specify 50000 partitions but from Gateway to broker it would contain just 50).","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/safeguard-topic-creation-validation/"},"content":[{"nodeType":"text","value":"Download","marks":[],"data":{}}]},{"nodeType":"text","value":" this Interceptor from Conduktor Marketplace.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor Gateway is a stateless Java application that presents itself as a regular Kafka cluster to clients. Typical deployments have a Gateway cluster per Kafka cluster but a Gateway cluster can also be deployed per application based on use case. It's possible to have tiers of Gateway clusters where different levels of rules can be applied (i.e. rules for application 1, rules for application 2, and rules for the whole organization). ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7F4U8G04yrjeoAyTaiOpIo","type":"Asset","createdAt":"2023-05-17T11:35:49.701Z","updatedAt":"2023-05-17T11:35:49.701Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"in-blog diagram safeguard","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7F4U8G04yrjeoAyTaiOpIo/4e6b6000fd137846378f48ac15bd8462/Frame_60974.png","details":{"size":469105,"image":{"width":3916,"height":1200}},"fileName":"Frame 60974.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We released our open-source ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/platform/gateway/installation/opensource-install/"},"content":[{"nodeType":"text","value":"Gateway","marks":[],"data":{}}]},{"nodeType":"text","value":" and some pre-built ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/"},"content":[{"nodeType":"text","value":"Interceptors","marks":[],"data":{}}]},{"nodeType":"text","value":" for you to start trying Conduktor Gateway. As of now, we have the following Safeguarding capabilities available:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/safeguard-topic-creation-validation/"},"content":[{"nodeType":"text","value":"Topic creation validation","marks":[],"data":{}}]},{"nodeType":"text","value":" ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/safeguard-schema-validation/"},"content":[{"nodeType":"text","value":"Schema validation","marks":[],"data":{}}]},{"nodeType":"text","value":" ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/safeguard-read-only-topic/"},"content":[{"nodeType":"text","value":"Define topics as read-only","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/safeguard-limit-produce-size/"},"content":[{"nodeType":"text","value":"Producer throttling","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/safeguard-produce-validation/"},"content":[{"nodeType":"text","value":"Producer message checks","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/safeguard-alter-topics/"},"content":[{"nodeType":"text","value":"Alter topics","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/safeguard-join-group/"},"content":[{"nodeType":"text","value":"Limit","marks":[],"data":{}}]},{"nodeType":"text","value":" join group attempts","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/safeguard-limit-connections/"},"content":[{"nodeType":"text","value":"Limit","marks":[],"data":{}}]},{"nodeType":"text","value":" the number of connections per second","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"On top of these pre-built Interceptors, Conduktor Gateway provides a ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/platform/gateway/interceptors/write-an-interceptor/"},"content":[{"nodeType":"text","value":"development kit","marks":[],"data":{}}]},{"nodeType":"text","value":" for making new interceptors specific to your use cases. Interceptors can be created by implementing a few interfaces, an open-source reference edition of the Gateway is provided for testing and running these.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In this article, we discussed how safeguarding can be used to prevent outages and respond gracefully to change. Every organization is at a different stage of safeguarding. Achieving proactive safeguarding prevents outages and makes your Kafka clusters more resilient to unexpected change. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"However, there is a more subtle benefit to consider to proactive approaches when it comes to perception. Habit-based, and reactive approaches are typically seen by developers and operators as cumbersome barriers to running their applications but, due to the early alerting and specific nature of proactive safeguarding, this perception can be reset. With a proactive approach, it is much easier to get buy-in for technical best practices and governance from application teams.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor Gateway can help you get started on your journey with proactive safeguarding. This is a proven solution that powers one of our features (","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/kafka-playground-two-free-kafka-clusters-without-operational-hassles/"},"content":[{"nodeType":"text","value":"Conduktor Playground","marks":[],"data":{}}]},{"nodeType":"text","value":") today. It makes it easy to implement simple safeguarding rules and is easily extensible to meet the requirements of your organization.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You can ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/platform/gateway/installation/opensource-install/"},"content":[{"nodeType":"text","value":"download","marks":[],"data":{}}]},{"nodeType":"text","value":" the open-source version of Gateway from our Marketplace and start using ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/"},"content":[{"nodeType":"text","value":"Safeguard Interceptors","marks":[],"data":{}}]},{"nodeType":"text","value":" today. If you have enterprise use cases please ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/contact/"},"content":[{"nodeType":"text","value":"contact us","marks":[],"data":{}}]},{"nodeType":"text","value":" to experience Gateway Enterprise","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://conduktor-marketplace.vercel.app/contact/"},"content":[{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"27hnt3WWLz6zSByQ0XASIJ","type":"Entry","createdAt":"2023-05-16T14:20:00.654Z","updatedAt":"2023-05-16T14:20:00.654Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Tom Scott","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3QYOKzEhc5XaZ9cha3B9H2","type":"Asset","createdAt":"2023-05-16T14:19:52.241Z","updatedAt":"2023-05-16T14:19:52.241Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Tom Scott picture","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3QYOKzEhc5XaZ9cha3B9H2/2adb8beb50205a667b486a7b55cd51c5/1653936876979.jpeg","details":{"size":7145,"image":{"width":200,"height":200}},"fileName":"1653936876979.jpeg","contentType":"image/jpeg"}}}}},"publishDate":"2023-05-17T00:00-04:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7BZ5f0AUS0Q5c0oE8vSEa5","type":"Entry","createdAt":"2023-05-16T02:20:45.625Z","updatedAt":"2023-05-16T02:22:47.541Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Enhance Apache Kafka with Conduktor Gateway","description":"We're excited to launch Conduktor Gateway on our Marketplace to help teams test, govern, and secure Apache Kafka","noIndex":false}},"slug":"enhance-apache-kafka-with-conduktor-gateway","title":"Enhance Apache Kafka with Conduktor Gateway","navbarDescription":"Enhance Apache Kafka with Conduktor Gateway","description":"We're excited to launch Conduktor Gateway on our Marketplace to provide central platform teams responsible for Kafka with enhanced functionality. Enforce best practices, improve your governance, and reduce your cost.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"20zxsEMWgvDt6W3sQYudTl","type":"Asset","createdAt":"2023-05-15T22:46:16.629Z","updatedAt":"2023-05-15T22:46:16.629Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Gateway/Marketplace","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/20zxsEMWgvDt6W3sQYudTl/79e0b18fad1e4d9bd74196b44657c61e/Gateway.png","details":{"size":594988,"image":{"width":2048,"height":1080}},"fileName":"Gateway.png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka is a highly available, and scalable data streaming platform that thousands of companies use to support their mission-critical applications. While it provides immense power to teams delivering real-time applications, there are limitations in Kafka that lead to time sinks for central platform and development teams. We're excited to launch Conduktor Gateway to provide central platform teams responsible for Kafka with enhanced functionality to help govern, secure, and enable Kafka self-service to teams across an organization.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"At a fundamental level, ","marks":[],"data":{}},{"nodeType":"text","value":"Conduktor Gateway","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" is the transport layer between your Kafka client applications and your Kafka clusters. This transport layer is enhanced with ","marks":[],"data":{}},{"nodeType":"text","value":"Conduktor Interceptors","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" or plugins that you can install on Gateway.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Announcing Conduktor Marketplace","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To ensure people can have easy access to both the Gateway and Interceptors we're announcing the release of ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io"},"content":[{"nodeType":"text","value":"Conduktor Marketplace","marks":[],"data":{}}]},{"nodeType":"text","value":"! Our marketplace contains an open-source version of Conduktor Gateway along with pre-built Interceptors or plugins that help you enhance Kafka for free. You'll also be able to build Interceptors to introduce any functionality you feel is missing in your Kafka.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/platform/gateway/installation/opensource-install/"},"content":[{"nodeType":"text","value":"Download open-source Gateway","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Apache Kafka Limitations and Gateway Interceptors","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor has been used by developers and central platform teams to explore and troubleshoot issues in Kafka data. Our customers operating Kafka at scale have increasingly had pain points surrounding governance, testing, and cost. With the launch of Conduktor Marketplace, we're releasing our first batch of pre-built Interceptors that are going to help solve the most critical issues for teams using Kafka.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Enforcing Kafka best practices across the organization","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Central platform teams often have the most Kafka expertise in an organization. When Kafka starts to spread to individual project teams these central platform teams are often responsible for enabling these teams with Kafka best practices.","marks":[],"data":{}}]},{"nodeType":"heading-4","data":{},"content":[{"nodeType":"text","value":"Introducing Safeguard","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor Safeguard protects your data and infrastructure from Kafka errors by enforcing rules that are centrally defined. Rules can be defined by the central platform team when they should be followed by the whole organization or defined by the individual domain teams when that expertise is important. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Safeguard Interceptors:","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/safeguard-topic-creation-validation/"},"content":[{"nodeType":"text","value":"Manage cost and data resilience","marks":[],"data":{}}]},{"nodeType":"text","value":" by ensuring that topic creation adheres to required minimum and maximum values for replication factor and partition count.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/safeguard-schema-id-present/"},"content":[{"nodeType":"text","value":"Reduce outages","marks":[],"data":{}}]},{"nodeType":"text","value":" by ensuring that all records produced in Kafka have a schema set.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/safeguard-read-only-topic/"},"content":[{"nodeType":"text","value":"Ensure data quality","marks":[],"data":{}}]},{"nodeType":"text","value":" in Kafka by defining topics as read-only. ","marks":[],"data":{}}]}]}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Increasing the reliability of Kafka","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka is a resilient system so it is difficult to simulate and test application problems that could occur in production in your dev and testing environments.","marks":[],"data":{}}]},{"nodeType":"heading-4","data":{},"content":[{"nodeType":"text","value":"Introducing Chaos","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With Conduktor Chaos you can simulate incidents in your Kafka application's development or testing environments. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Chaos Interceptors:","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ensure that your Kafka applications can ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/chaos-message-corruption/"},"content":[{"nodeType":"text","value":"handle messages","marks":[],"data":{}}]},{"nodeType":"text","value":" which don't arrive in the format that is expected. For example, how would your Kafka application react if it's expecting a message similar to ","marks":[],"data":{}},{"nodeType":"text","value":"transferAmount:10, currency: pounds","marks":[{"type":"code"}],"data":{}},{"nodeType":"text","value":" but actually receives ","marks":[],"data":{}},{"nodeType":"text","value":"transferAmount: 10, currency: poundsABC?","marks":[{"type":"code"}],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/chaos-duplicate-writes/"},"content":[{"nodeType":"text","value":"Validate","marks":[],"data":{}}]},{"nodeType":"text","value":" that your application behaves correctly when duplicate records are written to Kafka.","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Gateway Enterprise","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Our pre-built Interceptors in Conduktor Marketplace give you a preview of the kind of functionality that you can build into Kafka. Gateway Enterprise has Interceptors that are more in line with the needs of companies that are operating Kafka at scale.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Ensuring security and compliance in Kafka","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Encrypting the data flowing through Kafka either requires you to implement encryption at the application layer before the data is sent or after it is received in Kafka. This either requires you to spend cycles on your development team to build encryption into all of your applications or time from your central platform teams to implement encryption in storage.","marks":[],"data":{}}]},{"nodeType":"heading-4","data":{},"content":[{"nodeType":"text","value":"Introducing Encryption","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/dataMasking-data-masking/"},"content":[{"nodeType":"text","value":"Conduktor Encryption","marks":[],"data":{}}]},{"nodeType":"text","value":" works after data is sent from your application before it reaches your Kafka brokers. You can encrypt messages down to the field level using KMS solutions from AWS, Azure, or GCP.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Reducing the cost of Kafka","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If you're in a company where you have a large number of clusters for historical, technical, or other reasons, these clusters cost you money, whether they're physical or in the cloud.","marks":[],"data":{}}]},{"nodeType":"heading-4","data":{},"content":[{"nodeType":"text","value":"Introducing Multi-tenancy","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Similar to virtual machines, ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/interceptors/multiTenancy-multi-tenancy/"},"content":[{"nodeType":"text","value":"Conduktor Multi-tenancy","marks":[],"data":{}}]},{"nodeType":"text","value":" enables you to create as many virtual clusters as you need on a single Kafka cluster, allowing you to mathematically reduce the number of clusters you need to pay and maintain. ","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This is just the start of what we're building with Conduktor Gateway. We'll continue to release more Interceptors based on feedback from our customers. If you have any ideas, please let us ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/contact/"},"content":[{"nodeType":"text","value":"know","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Today, users have the choice of ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/platform/gateway/installation/opensource-install/"},"content":[{"nodeType":"text","value":"downloading","marks":[],"data":{}}]},{"nodeType":"text","value":" the open-source version of Gateway from our Marketplace to start enhancing Kafka with our free Interceptors or ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://marketplace.conduktor.io/contact/"},"content":[{"nodeType":"text","value":"contacting","marks":[],"data":{}}]},{"nodeType":"text","value":" us to experience Gateway Enterprise","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://conduktor-marketplace.vercel.app/contact/"},"content":[{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6V3UfdQUX4TeDpfFYtFD6I","type":"Entry","createdAt":"2023-02-22T19:20:39.991Z","updatedAt":"2023-02-22T19:20:39.991Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Shashank","surname":"Golla","title":"Head of Product Marketing","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"52ohL6VuQifd4USoHGV1zX","type":"Asset","createdAt":"2023-02-22T19:20:34.399Z","updatedAt":"2023-02-22T19:20:34.399Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"1586986866218","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/52ohL6VuQifd4USoHGV1zX/c88afc8b9228f4abef654dfd09be6996/1586986866218.jpeg","details":{"size":34960,"image":{"width":800,"height":800}},"fileName":"1586986866218.jpeg","contentType":"image/jpeg"}}}}},"publishDate":"2023-05-16T00:00-04:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7tQoWpAmstir7pfL9O4bk9","type":"Entry","createdAt":"2023-05-11T14:09:35.963Z","updatedAt":"2023-05-11T14:09:35.963Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"The story of Conduktor Gateway","description":"The personal journey that Florent Ramiere took to build Conduktor Gateway","noIndex":false}},"slug":"the-story-of-conduktor-gateway","title":"The story of Conduktor Gateway","navbarDescription":"The personal journey that Florent Ramiere took to build Conduktor Gateway","description":"In this article, we discuss why Conduktor felt now is the right time to introduce Conduktor Gateway to the world. Florent Ramiere shares his journey with Kafka from being a beginner to completing his first Kafka projects, his first mistakes, and how his approach to Kafka changed throughout this journey.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"30JZD3XsTj8gDnWFHfwlaS","type":"Asset","createdAt":"2023-05-11T17:12:45.689Z","updatedAt":"2023-05-11T17:12:45.689Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Conduktor story hero image","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/30JZD3XsTj8gDnWFHfwlaS/d048b0642a622cf28a2b093dab25ea50/image__9_.png","details":{"size":1834786,"image":{"width":2048,"height":1080}},"fileName":"image (9).png","contentType":"image/png"}}},"richText":{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"Hello, my name is Florent Ramière, and I am one of the authors of Conduktor Gateway. In this blog post, I want to present the history of Gateway and how I came to the realization that Apache Kafka was not complete and needed further development.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"My beginnings with Kafka","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"Like many others, I entered a project in the middle of development when Kafka was already introduced by a predecessor. I was working in the banking sector in an exciting and fast-paced environment full of new technologies and highly skilled people. When I started, Kafka was only used as a high-quality buffer for collecting large amounts of logs and metrics.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"With our small team, we had to implement monitoring, alerting, and processing using technologies such as Storm, then Spark, and eventually Kafka Streams. As we gained experience, I quickly acquired the necessary skills to perform my job, and everything seemed to be going well.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"However, I soon discovered that I didn't know Kafka as well as I thought. I only knew what was absolutely necessary to do my job, but not much more. The reality of production issues and managing various Kafka requirements forced me to re-evaluate my knowledge a bit too late.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"This situation is the root of many IT problems: addressing the most pressing issues and then moving on. Although I was considered the local Kafka expert, there was still so much I didn't know. I had to learn from my mistakes and correct numerous errors, most of which were avoidable.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"This led me to dive deeper into Kafka and its ecosystem. My growing interest in Kafka and its potential prompted me to join Confluent, the company behind Apache Kafka.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"My beginnings at Confluent","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"As a Customer Success Technical Architect at Confluent, I was responsible for supporting major clients across Europe. With the many lessons learned from my previous experience, I worked with various clients, each with different levels of maturity and unique contexts, every day I was discovering new ways to encounter challenges with Kafka.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"I wanted to share my experience with a broader audience, so I began giving talks at conferences on Kafka patterns and anti-patterns. I presented the problems that some people had already encountered, as well as those they hadn't yet faced but were likely to experience during their journey with Kafka.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"In essence, I was fixing problems after Kafka had been integrated. I decided to shift my reactive approach to a more proactive one, joining the pre-sales team. The goal was to set up teams for success from the very beginning, helping them achieve their business objectives.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Despite good intentions, I observed that Kafka implementations were often plagued by production bugs or were reliant on people with experience. There were no automated elements to rely on, and governance was only as effective as the least informed person.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"In short, there was no simple way to:","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"address common configuration problems","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"address common bugs or development issues","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"simulate real-world scenarios to ensure application resilience","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"enforce rules where they matter","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"do field-level encryption","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"do multi-tenancy or namespacing","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"do RBAC","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"do proper auditing","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"send large messages","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"reduce networking costs","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"add real lineage","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"and much much more","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"}],"nodeType":"unordered-list"},{"data":{},"content":[{"data":{},"marks":[],"value":"The arrival of cloud technology required me to learn and stay updated on various subjects like connectivity, VPC/VNet peering, and private links of the world. The non-transitivity of peering networks and Kafka is a significant issue. One way to address it is by using proxies that can natively talk to the Kafka protocol.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"At that time, I began to see the Kafka protocol as a means of solving networking or security problems, recommending and implementing Grepplabs' ","nodeType":"text"},{"data":{"uri":"https://github.com/grepplabs/kafka-proxy"},"content":[{"data":{},"marks":[],"value":"Kafka-proxy","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":" for our clients.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Things progressed rapidly, and I discovered that interest in proxies within the Kafka universe was booming. On one hand, Apache Pulsar was developing ","nodeType":"text"},{"data":{"uri":"https://github.com/streamnative/kop"},"content":[{"data":{},"marks":[],"value":"Kafka on Pulsar","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":", a compatibility mode for using Pulsar with the Kafka protocol. On the other hand, Microsoft was also working on Kafka compatibility for ","nodeType":"text"},{"data":{"uri":"https://learn.microsoft.com/en-us/azure/event-hubs/azure-event-hubs-kafka-overview"},"content":[{"data":{},"marks":[],"value":"Event Hubs","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":". Other products like ","nodeType":"text"},{"data":{"uri":"https://redpanda.com/)"},"content":[{"data":{},"marks":[],"value":"Redpanda","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":" were starting to natively leverage the Kafka protocol.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"For me, the most significant catalyst was the discovery of the discussion about ","nodeType":"text"},{"data":{"uri":"https://github.com/envoyproxy/envoy/issues/2852"},"content":[{"data":{},"marks":[],"value":"Kafka support in Envoy","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":". Eye-opening!","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"A fascinating array of new features were being introduced to address Kafka's shortcomings, such as filters, rate limiting, transformations, fallback, validation, and more.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Gradually, I realized that these features provided concrete and unavoidable solutions to most of the problems I encountered daily with my clients. In short, these solutions addressed Kafka's limitations.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"I wanted to provide tangible solutions quickly to the teams I worked with daily. So, I decided to create this product myself.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Enter Conduktor","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"I was prepared to leave Confluent to start my own team, my own company, and wanted to help teams address the most common Kafka governance issues. As it happened, some people I had tried to recruit for Confluent contacted me and tried to get me to join their company. This company was Conduktor, which was building a desktop user interface for troubleshooting Apache Kafka that I had seen implemented by many clients and was a resounding success.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"The Conduktor team was in the process of expanding their existing products to create a web version and build testing and governance tools. We shared a common interest in governance. So, I decided to join this team to add an essential feature: the Gateway.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"To reverse the Kafka limitations above, with Conduktor Gateway, you'll be able to:","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"Create and enforce rules to address common configuration problems","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"Simulate real-world scenarios to address common bugs or ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"development issues before production","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"Encrypt data down to the field level without changing your applications","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"Have multiple virtual Kafka clusters on a single physical cluster","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"Define RBAC permissions for applications","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"Understand and audit who is doing what, when","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"and much much more","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"}],"nodeType":"unordered-list"},{"data":{},"content":[{"data":{},"marks":[],"value":"We'll be launching Conduktor Gateway at Kafka Summit 2023. So Conduktor won't only be the best user interface to troubleshoot and manage Kafka it'll also provide the critical features you feel are missing in Kafka to be successful.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"My journey with Kafka","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"As you can see, like everyone else, I have gradually matured through my experiences. I started as a beginner, completed my first projects, made my first mistakes, corrected them, and fundamentally reconsidered what Kafka is and how to approach it. I then worked on governance, change management, and ultimately focused on adding Kafka's missing features in a Gateway.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"I'm excited for us to release Gateway at Kafka Summit 2023 - if you're at Kafka Summit and want to see Gateway in action please book time with us ","nodeType":"text"},{"data":{"uri":"https://meetings.hubspot.com/conduktor/kafka-summit-london-may-2023-meeting-booking"},"content":[{"data":{},"marks":[],"value":"here","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":".","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"document"},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2SUuKYTftRBbrtanQNCM2U","type":"Entry","createdAt":"2023-05-11T13:22:23.157Z","updatedAt":"2023-05-11T13:22:23.157Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Florent Ramiere","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"GwiipkwALkxaRvxO12Hni","type":"Asset","createdAt":"2023-05-11T13:22:12.304Z","updatedAt":"2023-05-11T13:22:12.304Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Flo image","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/GwiipkwALkxaRvxO12Hni/fe71ddbd3be3b8d7ab93b119295f8788/1649405579476.jpeg","details":{"size":110643,"image":{"width":800,"height":800}},"fileName":"1649405579476.jpeg","contentType":"image/jpeg"}}}}},"publishDate":"2023-05-11T00:00-04:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1Aeghx8MrpZHvZIf0mvzED","type":"Entry","createdAt":"2023-04-25T08:05:02.375Z","updatedAt":"2023-04-26T15:29:10.480Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Key Learnings from Our 2023 Offsite","description":"CEO Nicolas Orban shares highlights and key learnings from our latest team gathering.","noIndex":false}},"slug":"key-learnings-2023-offsite","title":"Key Learnings from Our 2023 Offsite","navbarDescription":"This past March, the entire Conduktor team met outside of Paris for its fourth offsite event.","description":"This past March, the entire Conduktor team met outside of Paris for its fourth offsite event. CEO Nicolas Orban shares highlights and key learnings from our latest team gathering.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4YJN8ewIOP2wl65pyZkk7N","type":"Asset","createdAt":"2023-04-26T15:28:49.988Z","updatedAt":"2023-04-26T15:28:49.988Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"cover-color","description":"Conduktor offsite in Paris March 2023","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4YJN8ewIOP2wl65pyZkk7N/bf75737e48117664efee55e0731f9812/cover-color.png","details":{"size":1037079,"image":{"width":2048,"height":1080}},"fileName":"cover-color.png","contentType":"image/png"}}},"richText":{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"One of our fundamental values at Conduktor is “we learn”. We love constantly striving to improve, and I wanted to share how we put this into practice for our team when hosting our fourth company offsite. As a distributed company with employees based primarily in the UK, France, and the US, our team predominantly works remotely or in a hybrid manner. This year we gathered our team in Cély, on the outskirts of Paris. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Bringing a distributed team together provides numerous benefits, including a chance to kick off projects, celebrate wins, and encourage team cohesion. However, I think it is important to think about the purpose and goal of each offsite individually. For our fourth event, we could easily have followed our previous patterns and schedules and organized something very similar where the focus had been on packing in as much as possible.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Although for this offsite, we wanted something different. A change in pace and direction was needed for our team. The focus was not on cramming as many sessions, workshops and meetings in as we could. Instead, we really wanted to make sure that after a particularly turbulent ride for tech over the last year, we strengthened the communication and understanding of our strategy and vision, and gave our team a renewed sense of energy, purpose, and motivation for the rest of 2023. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"28Z8sf9twASDDAYrT1zbRx","type":"Asset","createdAt":"2023-04-26T15:30:26.873Z","updatedAt":"2023-04-26T15:30:26.873Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"group-image1","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/28Z8sf9twASDDAYrT1zbRx/581e5159db866fdb6fd339b98ecbb7ff/group-image1.png","details":{"size":665428,"image":{"width":1816,"height":945}},"fileName":"group-image1.png","contentType":"image/png"}}}},"content":[],"nodeType":"embedded-asset-block"},{"data":{},"content":[{"data":{},"marks":[],"value":"Key learnings","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"Focusing on quality over quantity ","nodeType":"text"}],"nodeType":"heading-3"},{"data":{},"content":[{"data":{},"marks":[],"value":"Previous offsites have traditionally been a week-long, and though this might have provided us more time, in theory, to be together as a team, I think it’s safe to say that a week can overstretch people. Weekend travel, time away from other commitments, and jet lag can all contribute to a team feeling frustrated. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"For this offsite, we aimed to get the balance right. We spent three nights at the venue, and Monday and Thursday were dedicated to travel. Also, updating our toolkit to include ","nodeType":"text"},{"data":{"uri":"https://www.travelperk.com/"},"content":[{"data":{},"marks":[],"value":"TravelPerk","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":" allowed our employees to book their own travel, with clear guidelines and a budget, meaning they were able to build more flexibility into their plans and manage their own time. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4idwfGX1oQGWY3SuTl1q0Z","type":"Asset","createdAt":"2023-04-26T15:31:41.953Z","updatedAt":"2023-04-26T15:31:41.953Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"gallery1","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4idwfGX1oQGWY3SuTl1q0Z/8a332b2476fc328ddfdd4753e6f90133/gallery1.png","details":{"size":548020,"image":{"width":1258,"height":1309}},"fileName":"gallery1.png","contentType":"image/png"}}}},"content":[],"nodeType":"embedded-asset-block"},{"data":{},"content":[{"data":{},"marks":[],"value":"Catering to different needs","nodeType":"text"}],"nodeType":"heading-3"},{"data":{},"content":[{"data":{},"marks":[],"value":"We have stayed at some incredible venues at our offsites but finding one that suits the needs of a growing team means changing our requirements every time. For previous offsites, our activities mainly focused on sports. Not everyone though is a sports fan, and with a diverse team, it’s important to offer an inclusive range of activities that can be enjoyed by everyone. ","nodeType":"text"},{"data":{"uri":"https://www.chateauform.com/en/"},"content":[{"data":{},"marks":[],"value":"Chateauform","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":" was able to offer varied activities including an Escape Room, a cooking challenge, and an Olympic Games with a twist. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Along with offering this variety, we made sure that our teams for each “challenge” or activity were deliberately mixed up so folks from different countries, departments and seniority levels all got the chance to work on something together; in many cases for the first time. This was truly great to see, and already I am seeing the benefits of bringing our people together and the collaboration and projects that have come out of this since. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"666rSjYd7KEd04SmsSfxzK","type":"Asset","createdAt":"2023-04-26T15:32:04.438Z","updatedAt":"2023-04-26T15:53:04.167Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"gallery2","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/666rSjYd7KEd04SmsSfxzK/f5f902af25f1fb79a5fea525c2ab4a73/gallery2__1_.png","details":{"size":582301,"image":{"width":1258,"height":1309}},"fileName":"gallery2 (1).png","contentType":"image/png"}}}},"content":[],"nodeType":"embedded-asset-block"},{"data":{},"content":[{"data":{},"marks":[],"value":"Keep calm and communicate ","nodeType":"text"}],"nodeType":"heading-3"},{"data":{},"content":[{"data":{},"marks":[],"value":"It’s impossible to predict months in advance all of the challenges that might arise when planning a company offsite. It’s safe to say though we didn’t expect our Paris event to fall simultaneously with nationwide strikes and protests against France’s pension reform. Our People team worked incredibly hard to make sure we had a contingency plan, and that risk and travel disruption were minimized. With train strikes in France, we weren’t even sure half the company would be able to get to our venue!","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"With an international team, it’s paramount to be mindful of different levels of knowledge and comfortability in travelling to areas that might be proving unpredictable. We ensured we had backup plans for all transport, and made ourselves comfortable with the fact we might even have to reschedule the whole event for another time/place. Our People team communicated this to our team transparently and clearly, and we adapted where necessary: changing some travel plans and meeting points. Luckily in the end we were able to go ahead with our offsite but if needed, we would have changed our entire plan. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[],"nodeType":"hr"},{"data":{},"content":[{"data":{},"marks":[],"value":"These are just some of the learnings I have taken from hosting our fourth offsite event, and personally, I have been left energized, inspired, and eager for our fifth offsite! I hope our employees have felt the same, and if our post-offsite survey is anything to go by, it seems they did. 100% of the team rated the event 4 out of 5 or above, and we have also found a notable increase in our team’s self-identified confidence and knowledge of our products and strategy for 2023 since hosting. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"If you are interested in joining us for our next offsite find our careers page ","nodeType":"text"},{"data":{"uri":"https://www.conduktor.io/careers/"},"content":[{"data":{},"marks":[],"value":"here","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":" with all our current open positions. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3dcvxSshBIJfQECuUoDguV","type":"Asset","createdAt":"2023-04-26T15:31:05.510Z","updatedAt":"2023-04-26T15:31:05.510Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"group-image2","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3dcvxSshBIJfQECuUoDguV/8919153c67c799d124ae3ba47be66bfe/group-image2.png","details":{"size":498995,"image":{"width":1816,"height":945}},"fileName":"group-image2.png","contentType":"image/png"}}}},"content":[],"nodeType":"embedded-asset-block"},{"data":{},"content":[{"data":{},"marks":[],"value":"","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"document"},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"39amvlQKvYoiwNy5AjnaC1","type":"Entry","createdAt":"2021-10-08T17:39:21.943Z","updatedAt":"2022-06-28T17:56:13.133Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Nicolas","surname":"Orban","title":"Chief Executive Officer","github":"https://github.com/conduktor","linkedIn":"https://www.linkedin.com/in/nicolas-orban-64b660b8/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6dM5mpFwwNxnHFqoqq1Zlt","type":"Asset","createdAt":"2021-10-08T17:39:18.479Z","updatedAt":"2022-09-25T07:25:35.376Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Nicolas Orban","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/6dM5mpFwwNxnHFqoqq1Zlt/dbce1134994dda1f70456bca598f0f18/nicolas.png","details":{"size":45969,"image":{"width":192,"height":192}},"fileName":"nicolas.png","contentType":"image/png"}}}}},"publishDate":"2023-05-01T00:00-04:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"386Hgzb2Doy0Jc1k2vaH6e","type":"Entry","createdAt":"2023-05-01T16:24:03.744Z","updatedAt":"2023-05-01T16:24:03.744Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"What is a Data Mesh? | Conduktor Blog","description":"Data mesh is a paradigm that decentralizes data ownership and governance, enabling data domains to share and collaborate.","keywords":["data mesh","data mesh architecture","what is data mesh","data architecture principles"],"noIndex":false}},"slug":"data-mesh","title":"What is a Data Mesh?","navbarDescription":"Data mesh is a paradigm that decentralizes data ownership and governance.","description":"Data mesh is a paradigm that decentralizes data ownership and governance, enabling data domains to share and collaborate.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6tToGHCjgGNhwKNDnY9Kyt","type":"Asset","createdAt":"2023-05-02T23:20:13.008Z","updatedAt":"2023-05-04T07:38:11.917Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Data Mesh Cover","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/6tToGHCjgGNhwKNDnY9Kyt/aa548956b71275123c147e7c9e6be293/data-mesh-cover.png","details":{"size":305929,"image":{"width":2048,"height":1080}},"fileName":"data-mesh-cover.png","contentType":"image/png"}}},"richText":{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"Data mesh is a modern approach to managing data in large organizations that aims to increase agility, autonomy, and scalability by decentralizing data ownership. The idea of data mesh was first proposed by Zhamak Dehghani, a software architect at ThoughtWorks, ","nodeType":"text"},{"data":{"uri":"https://martinfowler.com/articles/data-mesh-principles.html"},"content":[{"data":{},"marks":[],"value":"in a blog post published in May 2019","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":". Since then, the concept has gained significant traction among data and technology leaders who recognize the limitations of traditional centralized data management approaches. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"The data mesh concept shares many similarities with domain-driven design (DDD) and Microservices principles. Both of these approaches prioritize the autonomy of individual teams and the modularization of complex systems. Data mesh is a paradigm shift from the traditional centralized data warehouse or lake approach, where data is collected and processed in a single location by a dedicated team. Data mesh advocates treating data as a product that can be owned, governed, and delivered by different domain teams across the organization. This way, data can be more accessible, reliable, and relevant for various use cases and stakeholders.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"The core principles of Data Mesh","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"Data mesh is based on the following core principles:","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Domain-oriented decentralized data ownership","nodeType":"text"},{"data":{},"marks":[],"value":" - Data should be owned by the domain experts who understand it best rather than by a central data team. This approach encourages autonomy and accountability and helps ensure that data is accurate, relevant, and up-to-date.","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Data as a product ","nodeType":"text"},{"data":{},"marks":[],"value":"- Data should be treated as a product that is designed, developed, and maintained by domain-oriented teams, with a focus on delivering value to internal and external customers. This approach fosters a product mindset and encourages a culture of experimentation, iteration, and continuous improvement.","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Federated data architecture","nodeType":"text"},{"data":{},"marks":[],"value":" - Data should be organized in a federated architecture, where data products are discoverable, reusable, and composable. This approach promotes modularity, interoperability, and scalability, enabling teams to collaborate and share data effectively.","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Self-serve data infrastructure","nodeType":"text"},{"data":{},"marks":[],"value":" - Teams should have access to self-serve data infrastructure to easily discover, access easily, and process data. This approach reduces dependencies on centralized data teams and empowers domain-oriented teams to quickly and independently create and iterate on data products.","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"}],"nodeType":"unordered-list"},{"data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5RhlKFHdgyJaED6U01bb1M","type":"Asset","createdAt":"2023-05-02T23:21:30.940Z","updatedAt":"2023-05-04T07:38:54.605Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Data mesh - diagram 2","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5RhlKFHdgyJaED6U01bb1M/3b9715c3288e3a8e8c4436473f19d885/diagram2.png","details":{"size":113950,"image":{"width":1800,"height":1200}},"fileName":"diagram2.png","contentType":"image/png"}}}},"content":[],"nodeType":"embedded-asset-block"},{"data":{},"content":[{"data":{},"marks":[],"value":"Logical architecture of Data Mesh","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"The core principles mentioned above encourage us to visualize data mesh as a set of self-contained entities covered with a connecting tissue that shares the features in common. This understanding is quite different from the centralized monolithic architecture we observed in traditional data platforms. At a high level, the architecture consists of four main components:","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Domain-Oriented Teams: ","nodeType":"text"},{"data":{},"marks":[],"value":"These are teams that are responsible for developing and maintaining specific data products within their domain. Each team has a clear understanding of the business domain they serve and is empowered to make decisions about the data products they develop.","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Data Products:","nodeType":"text"},{"data":{},"marks":[],"value":" These are self-contained, modular pieces of data infrastructure that are developed by domain-oriented teams. Each data product is designed to solve a specific business problem and is responsible for its own data quality, governance, and delivery.","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Platform Services:","nodeType":"text"},{"data":{},"marks":[],"value":" These are services that provide infrastructure and support for data products, such as data storage, compute, and security. Platform services are owned and managed centrally to ensure consistency across domains.","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Data Mesh Bus:","nodeType":"text"},{"data":{},"marks":[],"value":" This is a shared communication layer that enables data products to communicate with each other and with external systems. The data mesh bus allows data products to be composed and orchestrated to solve more complex business problems.","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"}],"nodeType":"ordered-list"},{"data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"74G7LNH5VtjbJ8Qaa0s13G","type":"Asset","createdAt":"2023-05-02T23:21:58.076Z","updatedAt":"2023-05-04T07:39:12.485Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Data mesh - diagram 3","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/74G7LNH5VtjbJ8Qaa0s13G/b0ca39aab2a8692206f095bf1c141928/diagram3.png","details":{"size":85351,"image":{"width":1800,"height":1745}},"fileName":"diagram3.png","contentType":"image/png"}}}},"content":[],"nodeType":"embedded-asset-block"},{"data":{},"content":[{"data":{},"marks":[],"value":"Data mesh architecture is inherently distributed and decentralized. By decentralizing data ownership and management, domain-oriented teams can develop data products that better align with the needs of the business and deliver value faster. The platform services provide the necessary infrastructure to support data products, while the data mesh bus allows data products to be composed and orchestrated to solve complex business problems. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Domain-oriented data ownership and data pipelines","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"The term domain in data mesh represents individuals with a common business function—for example, sales, marketing, customer service, and more—providing more ownership to the data they produce. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Domain-oriented data ownership--a key pillar in data mesh, means that data is owned by the domain experts who understand it best. This approach promotes autonomy and accountability, ensuring data is accurate, relevant, and up-to-date. Domain-oriented teams own and develop data products that serve specific business needs, promote a product mindset, and focus on delivering value to internal and external customers.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"To support domain-oriented data ownership, data mesh promotes the use of data pipelines. Data pipelines move data from its source to its destination, a data product, or another system. These pipelines can be developed and managed by domain-oriented teams, which reduces dependencies on centralized data teams and promotes agility and autonomy.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Why Data Mesh?","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"Traditionally, the data platform of an organization was centered around a data warehouse, which was maintained by a central data team. These teams were small and composed of a handful of specialists. As the organizational data volume grows, these teams are often disrupted by fulfilling ever-increasing data requests. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"These traditional architectures were often slow to adapt to changing business needs and market conditions. Centralized data teams are often responsible for managing data infrastructure and developing data products, which can create bottlenecks and dependencies. This can lead to long development cycles, slow time-to-market, and an inability to respond quickly to changing business requirements. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Moreover, siloed data is another problem in traditional data architectures. The data is often siloed by department or function, which can lead to duplication of effort, inconsistencies in data, and a lack of collaboration and knowledge sharing. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Data mesh addresses these problems by promoting domain-oriented data ownership and developing domain-oriented data products. That encourages collaboration and knowledge sharing across domain-oriented teams and promotes creating reusable data products that can be shared and reused across the organization. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Do I need a Data Mesh?","nodeType":"text"}],"nodeType":"heading-3"},{"data":{},"content":[{"data":{},"marks":[],"value":"Whether or not your organization should start using a data mesh architecture depends on several factors, such as the size and complexity of your organization, the current state of your data architecture, and the availability of the necessary technical skills and resources.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Before adopting a data mesh architecture, it's important to consider the following critical factors:","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Business Needs:","nodeType":"text"},{"data":{},"marks":[],"value":" Consider whether the benefits of a data mesh architecture align with the business needs and objectives of your organization. For example, if your organization values autonomy and agility and operates in a dynamic and fast-paced environment, a data mesh architecture may be a good fit.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Organizational Structure: ","nodeType":"text"},{"data":{},"marks":[],"value":"Consider the current organizational structure of your company and how it aligns with a data mesh architecture. Data mesh requires domain-oriented teams with strong data expertise and a culture of collaboration and knowledge sharing. It's important to evaluate whether your organization has the right structure and culture to support a data mesh architecture.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Technical Skills and Resources: ","nodeType":"text"},{"data":{},"marks":[],"value":"Consider the technical skills and resources needed to implement and maintain a data mesh architecture. Data mesh requires a combination of technical skills, such as data engineering, data science, and DevOps, as well as the necessary technical infrastructure, such as cloud computing and big data technologies.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Data Governance: ","nodeType":"text"},{"data":{},"marks":[],"value":"Consider how a data mesh architecture will impact data governance. Data mesh requires strong data governance practices to ensure data quality, consistency, and security. Evaluating whether your organization has the necessary policies, processes, and tools to support data governance in a data mesh architecture is important.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Change Management:","nodeType":"text"},{"data":{},"marks":[],"value":" Consider how a data mesh architecture will impact your organization's culture and workflows. Adopting a data mesh architecture requires significant changes in managing and processing data. Planning for change management, including training and communication, is important to ensure successful adoption.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Why should data observability co-exist with a Data Mesh?","nodeType":"text"}],"nodeType":"heading-3"},{"data":{},"content":[{"data":{},"marks":[],"value":"Data observability is the practice of measuring, monitoring, and understanding the behavior of data in real-time to ensure that it is accurate, complete, and consistent. When you have a data mesh architecture in place, consider cultivating a good data observability practice. It ensures that the data products developed by domain-oriented teams are of high quality and meet the organization's needs.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"A solid data observability practice enables domain-oriented teams to monitor the quality and behavior of their data products in real-time, allowing issues and errors can be quickly identified and addressed before they become bigger problems. That also ensures data products are consistent and adhere to established data governance policies.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Another benefit is that it promotes transparency and trust. When domain-oriented teams have visibility into the behavior of their data products, they can share this information with other teams and stakeholders. This promotes collaboration and knowledge sharing and helps build trust in the data products developed by domain-oriented teams.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Data observability also enables organizations to improve the performance of their data products. By monitoring the behavior of data products in real time, organizations can identify bottlenecks and inefficiencies in their data pipelines. This information can then be used to optimize data pipelines and improve the performance of data products.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Benefits of Data Mesh","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Improved agility:","nodeType":"text"},{"data":{},"marks":[],"value":" Data mesh enables organizations to quickly adapt to changing business requirements and market conditions by promoting autonomy and decentralization. Domain-oriented teams can independently develop and iterate on data products, reducing dependencies on centralized data teams and increasing agility.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Better data quality:","nodeType":"text"},{"data":{},"marks":[],"value":" Data mesh promotes domain-oriented decentralized data ownership, which helps ensure that data is accurate, relevant, and up-to-date. This approach fosters a culture of accountability and encourages domain experts to take ownership of their data.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Faster time-to-market:","nodeType":"text"},{"data":{},"marks":[],"value":" Data mesh enables teams to quickly create and iterate on data products, reducing the time to market for new products and services. This approach also fosters a culture of experimentation and innovation, which can help organizations stay ahead of the competition.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Increased scalability: ","nodeType":"text"},{"data":{},"marks":[],"value":"Data mesh promotes a federated data architecture, which enables teams to share and reuse data products across the organization easily. This approach promotes modularity, interoperability, and scalability, enabling organizations to scale their data capabilities as needed.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Better collaboration: ","nodeType":"text"},{"data":{},"marks":[],"value":"Data mesh encourages collaboration and knowledge sharing across domain-oriented teams, reducing silos and promoting cross-functional teams. This approach also fosters a culture of transparency and openness, which can lead to better decision-making and increased trust.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Improved data governance: ","nodeType":"text"},{"data":{},"marks":[],"value":"Data mesh promotes governance at the domain level, which helps ensure that data is managed in a secure, compliant, and ethical manner. This approach also enables organizations to manage data privacy and regulatory compliance requirements better.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Conclusion","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"The rising significance of data mesh is driven by the need for organizations to manage large, complex data systems in a way that is scalable, flexible, and resilient. As more and more organizations adopt cloud-based architectures and microservices-based approaches, the need for a decentralized approach to data management becomes increasingly important.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"","nodeType":"text"},{"data":{"uri":"https://www.conduktor.io/kafka/what-is-apache-kafka/"},"content":[{"data":{},"marks":[{"type":"underline"}],"value":"Apache Kafka","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":" is a natural fit for the data mesh architecture because it provides a highly scalable, distributed, and fault-tolerant platform for managing streaming data. By using Kafka as the backbone of the data mesh bus, organizations can create a flexible, decentralized architecture that allows domain-oriented teams to own and manage their data products while still ensuring that data is available and accessible across the organization.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"","nodeType":"text"},{"data":{"uri":"https://www.conduktor.io/"},"content":[{"data":{},"marks":[{"type":"underline"}],"value":"Conduktor","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":" is an enterprise-grade platform for Apache Kafka that provides tools for monitoring, managing, and developing Kafka-based applications. We provide a centralized platform for managing Kafka-related tasks related to the data mesh, including real-time monitoring of Kafka clusters and topics, schema management for Kafka data, and tools for managing Kafka-based workflows, while still allowing individual domain-oriented teams to own and manage their data products. This aligns with the data mesh philosophy of decentralized data ownership and management, while still providing the necessary infrastructure and governance to ensure data quality and security.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Conduktor's Topic as a service (TaaS) crystalizes this philosophy by giving developers from individual teams the power to create, update, share, and promote Kafka resources with the automated blessing of the central platform team.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Ready to get started? ","nodeType":"text"},{"data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"data":{},"marks":[],"value":"Start your free trial today","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":"!","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"document"},"publishDate":"2023-05-04T00:00-04:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5bzehaMtE9P1EdwRS8pn0y","type":"Entry","createdAt":"2023-04-25T16:19:14.620Z","updatedAt":"2023-04-28T12:39:31.394Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Security for humans and applications in Apache Kafka","description":"We'll discuss RBAC and ACL, the two most viable ways to protect and provide access to resources inside of your Apache Kafka deployment.","noIndex":false}},"slug":"security-rbac-acl-apache-kafka","title":"Security for humans and applications in Apache Kafka","navbarDescription":"What're the best strategies for Kafka auth?","description":"In this article, we will discuss the two most viable ways to protect and provide access to resources inside of your Apache Kafka deployment. In addition, we'll discuss how your choice of trusted or untrusted entities impacts your auth strategy.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7iaoQjVlThclKCFj87FJPm","type":"Asset","createdAt":"2023-04-25T14:17:40.792Z","updatedAt":"2023-04-25T14:17:40.792Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"RBAC Cover","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7iaoQjVlThclKCFj87FJPm/55ff1a0f7a2f3b9eb14f255248531cb7/RBACCover.png","details":{"size":71046,"image":{"width":2048,"height":1080}},"fileName":"RBACCover.png","contentType":"image/png"}}},"richText":{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"Allowing and denying access to a resource is a core requirement for any system and a major concern for any organization. This can be achieved by different methods including network access limitations, Access Control Lists(ACL), and Role-based access controls(RBAC). What is often forgotten is there are two different types of entities that need to be approved or denied; Humans and Applications. Inside of the human and application classification, there are also sub categories; trusted and untrusted.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"In this article, we will discuss the two most viable ways to protect and provide access to resources inside of your Apache Kafka deployment. In addition, we'll discuss how your choice of trusted or untrusted entities impacts your auth strategy.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Kafka authentication strategies","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"Network barriers","nodeType":"text"}],"nodeType":"heading-3"},{"data":{},"content":[{"data":{},"marks":[],"value":"Examples of network barriers include things like Firewalls, internal(non-routable) IPs, and air-gaping systems(Non-network connected). For a system like Kafka simply disconnecting from the network is simple but hard to scale for controlling user access. Because data distribution and sharing is a core use case for Kafka, firewall rules can quickly become complex and unmaintainable.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Access control lists","nodeType":"text"}],"nodeType":"heading-3"},{"data":{},"content":[{"data":{},"marks":[],"value":"The most basic type of access control is Access Control Lists(ACLs). ACLs are a simple list of rules that govern access to individual objects by individual entities. An example of a simple ACL list in Kafka would be:","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[{"type":"code"}],"value":"ANONYMOUS has NO ACCESS to topic=\"book_topic\";","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[{"type":"code"}],"value":"Bob has WRITE access to topic=\"book_topic\";","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[{"type":"code"}],"value":"Alice has READ access to topic=\"book_topic\";","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[{"type":"code"}],"value":"Mitch has READ AND WRITE access to topic=\"book_topic\";","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"blockquote"},{"data":{},"content":[{"data":{},"marks":[],"value":"In Kafka, these lists can be far more specific than simply allowing READ or WRITE access. Other objects such as consumer group creation, topic configuration management, cluster-level administration rights, and broker configuration operations are also in play and should be considered when creating and managing authorizations for your Kafka environments.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Access control lists can quickly become complex, hard to debug, and extremely large. In essence, teams are creating a substitute for RBAC that is hard to manage and scale. In order to solve this problem proper Role-based access controls should be introduced.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Role-based access controls (RBAC)","nodeType":"text"}],"nodeType":"heading-3"},{"data":{},"content":[{"data":{},"marks":[],"value":"Role-based access controls are an abstraction and tool to better organize ACL permissions. RBAC aims to solve the complexity and administration problems created by ACLs.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7qxzKmg7aqZWKUWnf9BNcj","type":"Asset","createdAt":"2023-04-25T14:20:41.797Z","updatedAt":"2023-04-25T14:20:41.797Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"RBAC Diagram 1","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7qxzKmg7aqZWKUWnf9BNcj/10854cb0f5895a59f6d3082d27d8d04f/RBACdiagram1.png","details":{"size":34811,"image":{"width":1800,"height":1200}},"fileName":"RBACdiagram1.png","contentType":"image/png"}}}},"content":[],"nodeType":"embedded-asset-block"},{"data":{},"content":[{"data":{},"marks":[],"value":"One of the core concepts of RBAC is that it allows you to create groups that are assigned permissions. Once you've created groups, you can simply add or remove users or applications to & from these groups to enable or disable access to objects within Kafka. This ability to define permissions once and then assign users based on the level of access they need allows teams to easily on-board and scale permissioning in an organization.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Integrating security on the control plane","nodeType":"text"}],"nodeType":"heading-3"},{"data":{},"content":[{"data":{},"marks":[],"value":"When implementing RBAC and ACL security mechanisms for an application with a user interface, ensuring consistent security practices for each end-user becomes a challenge. To maintain the same level of security, each end-user on the UI must have their own specific Kafka broker and identity, which can be difficult to manage and maintain, especially for a UI with a large user base.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"For a Kafka UI application, creating and managing individual clients for each user is cumbersome and resource-intensive. As the number of users grows, maintaining separate Kafka access for each user becomes increasingly impractical, leading to bottlenecks in performance and scalability.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Moreover, implementing distinct Kafka connections for each user can also hinder user experience, as it might require frequent updates and increase the complexity of managing user sessions. As a result, application developers need to strike a balance between maintaining a robust security posture and providing a seamless user experience.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Conduktor recommends a combination of both worlds: having the same RBAC source for both the control and data plane.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5mqPHODsB3EsmPfJuLSp7u","type":"Asset","createdAt":"2023-04-25T14:22:28.518Z","updatedAt":"2023-04-25T14:22:28.518Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"RBAC Diagram 2","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5mqPHODsB3EsmPfJuLSp7u/3466c0eab2506be2afd03b7a990d5d26/RBACdiagram2.png","details":{"size":62167,"image":{"width":1800,"height":2000}},"fileName":"RBACdiagram2.png","contentType":"image/png"}}}},"content":[],"nodeType":"embedded-asset-block"},{"data":{},"content":[{"data":{},"marks":[],"value":"The group is responsible for managing access to resources, and it has an association with identity and access control. Role-based access control (RBAC) is a method for restricting access based on the roles of users. The data plane is represented by Kafka, which is responsible for data processing, while the control plane is responsible for managing the data plane. Both data and control planes have an association with RBAC.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Summary","nodeType":"text"}],"nodeType":"heading-3"},{"data":{},"content":[{"data":{},"marks":[],"value":"RBAC should be considered the default choice for any organization no matter what stage of Kafka maturity they're at. The ability to define rules once and reuse them multiple times makes RBAC the obvious choice to reduce overhead and complexity when it comes to Kafka auth.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Conduktor sits on top of your existing Kafka ecosystem and helps you explore, manage, and govern Kafka data. Given that there is important customer data flowing through Conduktor, we implemented granular RBAC capabilities to define rules down to the topic level. You can also import your existing third-party auth source of truth directly into Conduktor.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"","nodeType":"text"},{"data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"data":{},"marks":[],"value":"Try","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":" Conduktor today with a 14-day free trial to see how it can help you accelerate your data streaming journey.","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"document"},"publishDate":"2023-04-28T00:00-04:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4y9OBheuV7UVfnAF69WvNR","type":"Entry","createdAt":"2022-10-19T11:49:16.968Z","updatedAt":"2022-10-19T12:12:48.643Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"The Conduktor Team in Milan","description":"In the final week of September, the entire Conduktor team gathered in Milan as we launched the Platform. Take a look at some of the best pictures.","noIndex":false}},"slug":"the-conduktor-team-in-milan","title":"The Conduktor Team in Milan","navbarDescription":"We brought the entire team together in Milan recently; check out the best pictures from the event!","description":"In the final week of September, the entire Conduktor team gathered in Milan as we launched the Platform. Take a look at some of the best pictures from the event.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2fNRHEdt36jdzb1cI20g0J","type":"Asset","createdAt":"2022-11-11T11:18:51.983Z","updatedAt":"2022-11-11T11:18:51.983Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"2 (1)","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2fNRHEdt36jdzb1cI20g0J/79c8ca8f192bd222a520f08b778dd342/2__1_.png","details":{"size":291901,"image":{"width":1024,"height":480}},"fileName":"2 (1).png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In the final week of September, the entire team gathered in Milan! We celebrated the launch of ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Conduktor Platform","marks":[],"data":{}}]},{"nodeType":"text","value":" and also were able to collaborate for one week and work on cultivating our values. It was an amazing way to get together and see the company's culture come to life. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"I have spoken already about how we used the offsite ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.linkedin.com/pulse/our-values-nicolas-orban/?trackingId=V5Ayw%2BlZQtiB5nC%2F6RN83A%3D%3D"},"content":[{"nodeType":"text","value":"as a chance to develop a new set of values","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" for Conduktor. Not only did we develop new values as a team, but we saw them in action as well all across the week that we were there.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This was our third offsite meeting, having previously gathered in Paris and Bordeaux. Milan didn't disappoint for a team that has more than quadrupled in size since the first offsite meeting. Collaboration is one of our new values and it was great to work on projects in person, with a company-wide competitive hackathon and plenty of team meetings. There were numerous opportunities to learn as well, with workshop sessions throughout the week.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6DHYy1Hxqk935t8czIz6dT","type":"Asset","createdAt":"2022-10-19T11:38:55.856Z","updatedAt":"2022-10-27T15:01:44.825Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"team-meeting","description":"Conduktor employees in a team meeting","file":{"url":"//images.ctfassets.net/lm7q36eaz111/6DHYy1Hxqk935t8czIz6dT/25c0114dc8b586fcf911dc181bcef77e/ced__1_.jpeg","details":{"size":184692,"image":{"width":1620,"height":1080}},"fileName":"ced (1).jpeg","contentType":"image/jpeg"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Of course, one of our new values was in evidence even before we arrived, as “we delivered” the launch of Conduktor Platform. But even in the company hackathon, we saw teams delivering impressive projects in very little time; the eventual winners were team “Wolfgang”, who created a new feature for the Platform called Radar. Keep your eyes peeled!","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Naturally, it wasn’t all work. There was plenty of time for some fun, with some particularly competitive games of volleyball:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2pW9dCRxkFZsng0OjY5LCp","type":"Asset","createdAt":"2022-10-19T11:43:27.263Z","updatedAt":"2022-10-28T13:26:27.673Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"volleyball","description":"Stephane takes the whole team on himself","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2pW9dCRxkFZsng0OjY5LCp/94b4c61f5f2dba2193d78850f4d43a3c/1__1_.jpeg","details":{"size":253742,"image":{"width":1920,"height":1080}},"fileName":"1 (1).jpeg","contentType":"image/jpeg"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"While in Bubble football the biggest challenge was to stay standing:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"73tdnPbLcUcwwMoYDafdzP","type":"Asset","createdAt":"2022-10-19T11:42:49.745Z","updatedAt":"2022-10-28T13:26:44.484Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"bubble-football","description":"An intrepid player makes a bold move","file":{"url":"//images.ctfassets.net/lm7q36eaz111/73tdnPbLcUcwwMoYDafdzP/4e07909cd4e057228a395d8809243dde/2__1_.jpeg","details":{"size":283966,"image":{"width":1920,"height":1080}},"fileName":"2 (1).jpeg","contentType":"image/jpeg"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"And a fair few of us found it in them to “be bold” and take a chance at Conduktor Karaoke:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1dgCyrmrWY6Bq8qJtQXNGO","type":"Asset","createdAt":"2022-10-19T11:44:05.232Z","updatedAt":"2022-10-28T13:27:26.378Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"karaoke","description":"Love at first sight for David and Nico","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1dgCyrmrWY6Bq8qJtQXNGO/8f72d931d7fc2b4facd1a0863026a306/3__1_.jpeg","details":{"size":271904,"image":{"width":1920,"height":1080}},"fileName":"3 (1).jpeg","contentType":"image/jpeg"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Capping off the week, we took the time to honor the Conduktor employees who best embodied our values and consistently went above and beyond to help deliver the successful launch of the Platform.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The winners were:","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Roch, Aine, Freddie, and Steven","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"tuohQCrbeC2y2L2WH2Dh6","type":"Asset","createdAt":"2022-10-19T11:45:33.226Z","updatedAt":"2022-10-28T13:27:38.415Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"prize-winners","description":"Roch, Aine, Freddie, and Steven show off their trophies","file":{"url":"//images.ctfassets.net/lm7q36eaz111/tuohQCrbeC2y2L2WH2Dh6/1b3fea0bdf5800fa2cd64c187d6daa06/4__1_.jpeg","details":{"size":265525,"image":{"width":1920,"height":1080}},"fileName":"4 (1).jpeg","contentType":"image/jpeg"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Thanks to everyone who made these past 6 months a great success. I’m looking forward to our next chance to bring everyone together.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If you want to join this great team and join us at our next offsite, we are hiring! Check out our ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/open-positions/"},"content":[{"nodeType":"text","value":"list of open positions","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" or get in touch directly if you feel you could be a great fit.","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"39amvlQKvYoiwNy5AjnaC1","type":"Entry","createdAt":"2021-10-08T17:39:21.943Z","updatedAt":"2022-06-28T17:56:13.133Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Nicolas","surname":"Orban","title":"Chief Executive Officer","github":"https://github.com/conduktor","linkedIn":"https://www.linkedin.com/in/nicolas-orban-64b660b8/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6dM5mpFwwNxnHFqoqq1Zlt","type":"Asset","createdAt":"2021-10-08T17:39:18.479Z","updatedAt":"2022-09-25T07:25:35.376Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Nicolas Orban","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/6dM5mpFwwNxnHFqoqq1Zlt/dbce1134994dda1f70456bca598f0f18/nicolas.png","details":{"size":45969,"image":{"width":192,"height":192}},"fileName":"nicolas.png","contentType":"image/png"}}}}},"publishDate":"2022-10-28T11:00+01:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5NID4vSYOVHo1ww7X7EA5G","type":"Entry","createdAt":"2023-03-17T20:48:40.477Z","updatedAt":"2023-03-19T11:59:52.578Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"What is Change Data Capture Anyways? | Conduktor Blog","description":" Learn what Change Data Capture (CDC) is, its use cases, and mechanisms.","noIndex":false}},"slug":"what-is-change-data-capture-anyways","title":"What is Change Data Capture Anyways?","navbarDescription":" Learn what Change Data Capture (CDC) is, its use cases, and mechanisms.","description":" Learn what Change Data Capture (CDC) is, its use cases, and mechanisms.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5OvdivWhT2aOQUWdWJQn3z","type":"Asset","createdAt":"2023-02-23T16:47:32.450Z","updatedAt":"2023-02-23T16:47:32.450Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"TAASDeepdive OG (1)","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5OvdivWhT2aOQUWdWJQn3z/03ec14228a7608f743bea2030e9cd159/TAASDeepdive_OG__1_.webp","details":{"size":8814,"image":{"width":1200,"height":630}},"fileName":"TAASDeepdive_OG (1).webp","contentType":"image/webp"}}},"richText":{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"The main reason why top companies don't stay at the top forever is because they can't keep up with new trends. Failure to notice new trends and patterns is caused by the inability to spot new changes in trend data and pattern adjustments. Making data driven decisions is not effective when data being used is not fresh and exhibits low quality. Industries change in split seconds and you have to know which change matters.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Real-time analytics and up-to-date data is the oxygen of effective decision making. Data that is not updated is dangerous and misleading when being used to plot the next technical move. Netflix stock was ","nodeType":"text"},{"data":{"uri":"https://time.com/6198354/netflix-subscribers-loss-second-quarter-future/#:~:text=Despite%20the%20jump%2C%20Netflix%20shares,investors%20about%20declining%20subscriber%20growth."},"content":[{"data":{},"marks":[{"type":"underline"}],"value":"down by 65% in 2022","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":" July and took a turn at the end of the year when it ","nodeType":"text"},{"data":{"uri":"https://variety.com/2023/tv/news/netflix-subscribers-earnings-q4-2022-1235493532/"},"content":[{"data":{},"marks":[{"type":"underline"}],"value":"gained 7 million subscribers","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":". It is important to know when data changes in order to invest well.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Data has to be updated even when it was changed a few seconds ago; there is a mechanism called Change Data Capture (CDC) that spots any data changes in the data source and triggers an update to the target. In this article, you will learn what Change Data Capture (CDC) is, its use cases, and mechanisms.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"What is Change Data Capture (CDC)","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"Change Data Capture (CDC) is a technology used to track and capture changes made to a database. It is used to replicate changes made to a source database to one or more target databases, in real-time. This technology is commonly used in data warehousing, business intelligence, and data integration scenarios.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"CDC works by monitoring the transaction logs of a source database. These logs contain a record of all changes made to the database, including insertions, updates, and deletions. CDC software reads these logs and captures the changes, which are then applied to the target database. This process is often referred to as \"log-based CDC.\"","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"There are two ways CDC occurs: either data change is pushed from the source to the data warehouse or a data change is pulled from the source by the target.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"It's easier to pull data changes from the source than it is to push data changes. This is because pulling data changes only requires the target to read the logs of the data source to identify any changes and take any actions if data has changed. Even though it is easy to pull data this comes with the problem of latency as the target has to consistently read the data source logs.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Pushing data occurs when the data source sends data changes to the target to take any necessary actions. There has to be a mechanism set up to receive and interpret the data changes. This approach comes with low latency.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"The only downside of this approach occurs when the target is not listening to the data source sending data changes. The target will miss out data changes. To solve this problem the target has to implement a queuing system that records the received data changes. The push approach is crucial for real-time data systems that need low-latency actions.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"What Are the Benefits of Using CDC?","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"There are several benefits of using CDC. One of the main advantages is that it allows for real-time data replication, which means that data changes made to the source database are quickly reflected in the target database. This is important for scenarios where the target database is used for reporting or analytics purposes, as it ensures that the data in the target database is always up-to-date.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"CDC also allows for greater flexibility in terms of data replication. With traditional data replication methods, the entire source database is typically replicated to the target database. With CDC, however, it is possible to replicate only specific tables or columns, which can be useful in scenarios where only a subset of the data is needed in the target database.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"CDC is also useful for data integration scenarios. It can be used to replicate data from multiple source databases to a single target database, which can then be used for reporting and analytics purposes. This can be a more efficient and cost-effective solution than using traditional data integration techniques, such as Extract, Transform, Load (ETL) processes.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Another benefit of CDC is that it can be used for data archiving. By capturing changes made to the source database, it is possible to create a historical record of the data. This can be useful for compliance purposes, as well as for troubleshooting and auditing.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Use Cases of CDC","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"\nReplication of data is one of CDC's most often used use cases. For backup and disaster recovery purposes, this approach is frequently used to duplicate data from one database to another, for instance, from a primary database to a secondary database. This can be carried out immediately, or on a set timeframe. By just collecting and copying changes made to the data, as opposed to the complete dataset each time, CDC enables efficient data replication. This increases replication speed and can significantly minimize the quantity of data that needs to be sent.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Keeping the data warehouse up to date is one of CDC's typical use cases. This method is used alongside data integration processes to gather, manage, and store massive amounts of data in a data warehouse for the purpose of reporting and analysis. With CDC, data can be gathered from several sources and compiled into a single data warehouse in real-time, close to real-time, or at a predetermined time. As a result, organizations are able to access all of their data consistently and make decisions based on that data.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"CDC can be utilized for auditing functions as well. It enables the keeping of a record of data changes, which can be advantageous for compliance and regulatory purposes. Additionally, it can be used to track modifications made by various users and spot any unwanted modifications. Security and fraud detection reasons may benefit from this.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"CDC can be utilized in a variety of various situations, including IoT, ETL, event sourcing, and others. Real-time monitoring and analysis of sensor data are made possible by the usage of CDC in the Internet of Things. CDC tracks and records changes made to sensor data in real-time. Data changes performed during the ETL (Extract, Transform, Load) process can be tracked and recorded using CDC, enabling reliable data replication and data warehousing. CDC can be used in event sourcing to track and record changes made to an application's state, enabling accurate tracking of the application's state and the capability to replay earlier states.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"CDC Methods and Mechanisms","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"CDC can be implemented in various ways. One common approach is to use a separate CDC software that reads the transaction logs of the source database and applies the changes to the target database. Another approach is to use the built-in CDC capabilities of the source database, such as ","nodeType":"text"},{"data":{"uri":"https://docs.oracle.com/goldengate/c1230/gg-winux/GGHDB/preparing-database-oracle-goldengate-cdc-capture.htm"},"content":[{"data":{},"marks":[{"type":"underline"}],"value":"Oracle GoldenGate","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":" or ","nodeType":"text"},{"data":{"uri":"https://learn.microsoft.com/en-us/sql/relational-databases/track-changes/enable-and-disable-change-data-capture-sql-server"},"content":[{"data":{},"marks":[{"type":"underline"}],"value":"SQL Server Change Data Capture","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":".","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"CDC can also be implemented using open-source solutions, such as ","nodeType":"text"},{"data":{"uri":"https://debezium.io/"},"content":[{"data":{},"marks":[{"type":"underline"}],"value":"Debezium","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":", which is a CDC platform built on top of Apache Kafka. Debezium allows for real-time data replication between a variety of different databases, including MySQL, PostgreSQL, and MongoDB.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[{"type":"bold"}],"value":"Here are different CDC Mechanisms you should Know:","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"The ","nodeType":"text"},{"data":{},"marks":[{"type":"bold"}],"value":"row versioning mechanism","nodeType":"text"},{"data":{},"marks":[],"value":" works by incrementing the version number when any change occurs on the table. The version number can start at 0 or 1. If the version number is 35 then you will know that the data in the table has been changed 35 times or 36 times if the version number started at 0. The row versioning mechanism is simple but it gets complicated to the target system when one record has multiple versions. Getting to know the latest version is difficult. But this can be solved by storing the last known version number in the references table ID. Checking the biggest version will give you the latest record version.","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"The ","nodeType":"text"},{"data":{},"marks":[{"type":"bold"}],"value":"Update timestamp","nodeType":"text"},{"data":{},"marks":[],"value":" is only concerned about giving you the latest record, not how many times the record has changed. This mechanism uses a timestamp to determine the latest version. The timestamp comprises date and time. The biggest timestamp resembles the latest version. This mechanism is not complex and does not need reference tables to solve the issue of a record having multiple versions. Every time data changes the timestamp is overwritten.","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"The ","nodeType":"text"},{"data":{},"marks":[{"type":"bold"}],"value":"Publish and subscribe queues","nodeType":"text"},{"data":{},"marks":[],"value":" mechanism uses the push approach as it sends the data changes to a queue where they will be retrieved by the target. This is good because it allows scalability; if many changes are made suddenly they can be stored in the queue and the target will scale them. Changing data sources using this system is easy because the data source and the target are decoupled by the middleman which is the queue.","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"},{"data":{},"content":[{"data":{},"content":[{"data":{},"marks":[],"value":"The","nodeType":"text"},{"data":{},"marks":[{"type":"bold"}],"value":" Database log scanners","nodeType":"text"},{"data":{},"marks":[],"value":" mechanism was built for two systems that have to be in sync especially in cases where the backup is being synchronized. This mechanism installs scanners on the database which log any changes being made to the data. These changes will trigger the backup to INSERT, UPDATE or DELETE depending on the modifications being made to the database. ","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"list-item"}],"nodeType":"unordered-list"},{"data":{},"content":[{"data":{},"marks":[],"value":"Limitations of Implementing CDC","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"The CDC technology has some restrictions. The fact that it can be resource-intensive is one of its key drawbacks. The reading of transaction logs and the updating of the target database can place a heavy burden on both the source and target databases. When a lot of data is being copied in real-time, this can be very difficult.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Another drawback of CDC is that it might be difficult to set up and run. It can take some time and a certain amount of technical knowledge to configure the CDC software, set up the replication process, and monitor the replication.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"Conclusion","nodeType":"text"}],"nodeType":"heading-2"},{"data":{},"content":[{"data":{},"marks":[],"value":"Change Data Capture (CDC) is a technology that allows for real-time data replication between a source and one or more target databases. It is commonly used in data warehousing, business intelligence, and data integration scenarios, but it can also be used for data archiving. CDC has many benefits, including real-time data replication, flexibility in terms of data replication, and efficient data integration. However, it also has some limitations, including resource-intensive process, and complexity of setup and maintenance.","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"In conclusion, Capture Data Change (CDC) is a potent tool that can be applied in a wide range of situations, such as auditing, data warehousing, and replication. In addition, it supports IoT, ETL, event sourcing, compliance, regulatory requirements, security, and fraud detection. It also provides for effective data replication. For enterprises trying to manage and interpret their data in real-time, it is an essential tool. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"One of the best tools you can use to implement CDC is Kafka which helps you to leverage CDC transaction log. But, Kafka is a powerful and sophisticated tool that needs to be managed. Don't worry about Kafka being too sophisticated. Conduktor gives you an intuitive UI that lets you manage your critical Kafka tasks, and it gives you monitoring and auditing features that help you observe your Kafka ecosystem. ","nodeType":"text"}],"nodeType":"paragraph"},{"data":{},"content":[{"data":{},"marks":[],"value":"","nodeType":"text"},{"data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"data":{},"marks":[{"type":"underline"}],"value":"Sign up for Conduktor today","nodeType":"text"}],"nodeType":"hyperlink"},{"data":{},"marks":[],"value":" for free to streamline and manage your Kafka CDC operations swiftly with our powerful, user-friendly platform!","nodeType":"text"}],"nodeType":"paragraph"}],"nodeType":"document"},"publishDate":"2023-03-17T08:00-05:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"136VGlHBGziOwFqXVhNxV4","type":"Entry","createdAt":"2022-11-16T14:25:01.973Z","updatedAt":"2023-03-07T08:57:39.229Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Security for Apache Kafka in Conduktor Platform","description":"A look at the Apache Kafka security features available for users of the Conduktor Platform., including data masking and encryption.","noIndex":false}},"slug":"security-for-apache-kafka-in-conduktor-platform","title":"Security for Apache Kafka in Conduktor Platform: A Deep Dive","navbarDescription":"A look at the Apache Kafka security features available for users of the Conduktor Platform.","description":"A look at the Apache Kafka security features available for users of the Conduktor Platform.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"zWrlLJyk64nPyiwgfvzsb","type":"Asset","createdAt":"2022-11-16T14:24:24.437Z","updatedAt":"2022-11-16T14:24:24.437Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"SecurityDeepDive","description":"Security for Apache Kafka in Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/zWrlLJyk64nPyiwgfvzsb/2acb2a3080006b6031da395f89794739/SecurityDeepDive.png","details":{"size":191249,"image":{"width":1920,"height":1080}},"fileName":"SecurityDeepDive.png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/introducing-the-conduktor-platform/"},"content":[{"nodeType":"text","value":"Conduktor Platform ","marks":[],"data":{}}]},{"nodeType":"text","value":"has introduced loads of impressive and innovative new solutions for Apache Kafka, but when it comes to importance, none will be as relevant as the security solutions. With so much of the world's infrastructure now involving Kafka in some way, enterprises need every tool at their disposal to help keep data safe. In this blog, we'll be looking at the security features available for users of the Conduktor Platform.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"How the Conduktor Platform provides security","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We can divide security features into two areas: security for your Kafka, and security for the Platform itself. There are three key Platform security features: ","marks":[],"data":{}},{"nodeType":"text","value":"Role-based access control (RBAC)","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":", ","marks":[],"data":{}},{"nodeType":"text","value":"Single sign-on (SSO)","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":", and ","marks":[],"data":{}},{"nodeType":"text","value":"audit logs","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". Security for Kafka is provided through ","marks":[],"data":{}},{"nodeType":"text","value":"End-to-end encryption","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" and ","marks":[],"data":{}},{"nodeType":"text","value":"Data masking","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"RBAC (Role Based Access Control)","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/rbac/"},"content":[{"nodeType":"text","value":"Role-based access control ","marks":[],"data":{}}]},{"nodeType":"text","value":"allows administrators to assign specific roles to individual users, specifying what each role can access. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In the Conduktor Platform, RBAC permissions can be set at the topic level, meaning different roles would have different levels of access to individual topics. You can assign multiple users to roles at once by creating groups.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let's imagine a standard Kafka dev team. One or more people will have \"Admin\" roles, having complete power over Conduktor Platform and the different roles. A \"Senior Developer\" role would have edit access to everything in the Platform, but no admin permissions like deleting users or editing roles. \"Junior Developer\" is more limited, without edit access to mission-critical topics. \"QA\" could be given access to Conduktor Testing and Conduktor Monitoring but not other parts of the Platform. And a \"Business User\" role could have read-only access to everything with no edit permissions.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2VFpDBGdleSMN8dMYHg2k5","type":"Asset","createdAt":"2022-11-16T14:06:40.471Z","updatedAt":"2022-11-17T14:38:20.217Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"newRBAC","description":"https://images.ctfassets.net/lm7q36eaz111/1P7jThZuMydb1kLILv58or/6200f4dc26d021d4244a7a192a2562cd/VideoThumbnail_Admin__1_.png","file":{"url":"//videos.ctfassets.net/lm7q36eaz111/2VFpDBGdleSMN8dMYHg2k5/811c6b2fe88a397c2b5c682a1fa8c951/newRBAC.mp4","details":{"size":1216915},"fileName":"newRBAC.mp4","contentType":"video/mp4"}}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"SSO (Single Sign On)","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/sso/"},"content":[{"nodeType":"text","value":"Single sign-on ","marks":[],"data":{}}]},{"nodeType":"text","value":"enables users to login to Conduktor Platform with a common set of credentials, for example, a Google or Microsoft account. The Conduktor Platform supports the LDAP and OpenId protocols for SSO. Once you've performed the necessary setup for these protocols, controlling access to the Platform is easy. Anyone with access to enterprise credentials will have access to Conduktor, and anyone you remove from your enterprise loses access. You'll have single, centralized control.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Having SSO is a necessary step in enabling RBAC, since it allows administrators to control registrations, instantly enabling or disabling users and assigning them to the correct roles.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5GIaBZWeGutKxTTK379kqU","type":"Asset","createdAt":"2022-11-16T14:10:31.596Z","updatedAt":"2022-11-16T14:10:31.596Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"SSO","description":"SSO in Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5GIaBZWeGutKxTTK379kqU/884aeedc27b7cb30c2a21bf971a4dddf/image_11_.png","details":{"size":248234,"image":{"width":1321,"height":836}},"fileName":"image(11).png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Audit Logs","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"An audit log is a track of actions taken within a system. Applications have always had logs to enable developers to fix bugs, but the purpose of an audit log is instead to see what actions users are taking and who is logged in. ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/audit/"},"content":[{"nodeType":"text","value":"Conduktor's audit logs ","marks":[],"data":{}}]},{"nodeType":"text","value":"are comprehensive, demonstrating everything a user does within the Kafka ecosystem. There are a number of reasons audit logs could be vital:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Compliance regulations affect a wide variety of industries. Audit logs include information about security for CIS, compliance with payment contracts (PCI DSS), and adherence to SOC 2.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Audit logs can be used to reconstruct the timeline of a system outage or incident and are used to differentiate between operator error and system errors.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If a security breach occurs, an audit log provides a clear trail showing what occurred, when it occurred, and who was responsible. For example, if personal data was leaked, breaching GDPR, you can track the flow of data through Kafka.","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With Conduktor Platform, you can see every action taking place within your Kafka ecosystem ensuring compliance and enhancing troubleshooting efforts. ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5NA0aQGysXJYCWm6YJE5Wy","type":"Asset","createdAt":"2022-11-16T14:11:17.030Z","updatedAt":"2022-11-17T14:38:45.120Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"auditLog","description":"https://images.ctfassets.net/lm7q36eaz111/Sr7bG0cLtNrJwxBpgVOQq/105f3ad9baa0b19f9106f570809ee539/VideoThumbnail_Console__1_.png","file":{"url":"//videos.ctfassets.net/lm7q36eaz111/5NA0aQGysXJYCWm6YJE5Wy/465befb0c3444538af4afd9bfaf07986/auditLog.mp4","details":{"size":3107830},"fileName":"auditLog.mp4","contentType":"video/mp4"}}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"End-to-end encryption","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Encryption is a challenge in Kafka. It doesn't exist as a standard feature of the technology. There are other security features like TLS, but TLS is used for transport layer security. It only protects your data as it's flowing across the network. TLS does not protect your data while on a Kafka client or broker. While there are workarounds - for example, you could encrypt the disks that your brokers are running on - they'll still have issues. Kafka is a distributed system moving messages from producers to consumers, meaning disk encryption only really works at one point. Anyone with access to the producers or consumers can see the unencrypted message. Until now, the only way to get faultless end-to-end encryption has been to make it yourself, with all the work that entails.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Conduktor Proxy, available to Conduktor Platform subscribers, can transparently encrypt and decrypt data as it is written or read from Kafka. This requires no changes to clients that access the proxy but retains the information in Kafka in a format that is unreadable by illegitimate clients. Implementation could not be simpler - a single API call can configure encryption and decryption for all data that passes through your clusters, or for specific topics.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Encryption can be applied to entire messages or sensitive fields within them. Plus, integration with Schema Registry supports a wide variety of message types.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Data Masking","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Even with encrypted messages, you don't necessarily want key holders to be able to view all the data contained within a message. For example, data protection regulations like GDPR & CCPA mean that personally identifiable information (PII) should not be viewable by just anyone. Conduktor's ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/data-masking/"},"content":[{"nodeType":"text","value":"data masking ","marks":[],"data":{}}]},{"nodeType":"text","value":"enables you to add instant, field-based masking across your entire Kafka ecosystem, meaning personal information stays personal.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4eLgTKwKFKGyHwlSodgpxa","type":"Asset","createdAt":"2022-11-16T14:12:10.568Z","updatedAt":"2022-11-17T14:38:51.918Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"dataMask","description":"https://images.ctfassets.net/lm7q36eaz111/Sr7bG0cLtNrJwxBpgVOQq/105f3ad9baa0b19f9106f570809ee539/VideoThumbnail_Console__1_.png","file":{"url":"//videos.ctfassets.net/lm7q36eaz111/4eLgTKwKFKGyHwlSodgpxa/acb07fd25415b36d213963c8c6ba77ab/dataMask.mp4","details":{"size":4281590},"fileName":"dataMask.mp4","contentType":"video/mp4"}}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Access Control Lists","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If you don't want to use RBAC and SSO, Kafka Access Control Lists (ACLs) are still supported by the Conduktor Platform. ACLs function similarly to RBAC, in that it determines what access an individual user has, but without the granularity of roles available through RBAC.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"How Do I Get Started?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Role-based access controls and audit logs will be enabled automatically when you have an enterprise subscription to Conduktor Platform. Single sign-on will require setup to connect to your enterprise credentials; social logins are automatic. Please see our ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://github.com/conduktor/conduktor-platform/tree/main/doc"},"content":[{"nodeType":"text","value":"documentation ","marks":[],"data":{}}]},{"nodeType":"text","value":"for more info on setting up.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"End-to-end encryption is run through the Conduktor Proxy, which is available on request to all enterprise subscribers: Please ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/contact/"},"content":[{"nodeType":"text","value":"contact us ","marks":[],"data":{}}]},{"nodeType":"text","value":"directly to find out how to gain access. To enable end-to-end encryption with the proxy, you need to run only a single command that defines encrypted fields. For example:","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7x7jawkDdyST4N7DxGw5dn","type":"Entry","createdAt":"2023-01-18T10:52:15.394Z","updatedAt":"2023-01-18T10:52:15.394Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"terminalBlock"}},"locale":"en-US"},"fields":{"command":"docker-compose exec kafka-client curl \\\n --silent \\\n --request POST \"conduktor-proxy:8888/tenant/1-1/feature/encryption\" \\\n --header 'Content-Type: application/json' \\\n --data-raw '{\n \"config\": { \n \"topic\": \"encrypted_topic\",\n \"fields\": [ { \n \"fieldName\": \"password\",\n \"keySecretId\": \"secret-key-password\",\n \"algorithm\": { \n \"type\": \"TINK/AES_GCM\",\n \"kms\": \"TINK/KMS_INMEM\" \n }\n },\n { \n \"fieldName\": \"visa\",\n \"keySecretId\": \"secret-key-visaNumber\",\n \"algorithm\": { \n \"type\": \"TINK/AES_GCM\",\n \"kms\": \"TINK/KMS_INMEM\" \n } \n }] \n },\n \"direction\": \"REQUEST\",\n \"apiKeys\": \"PRODUCE\"\n }'"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A full demo of encryption is ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://github.com/conduktor/conduktor-proxy-demos/tree/main/encryption"},"content":[{"nodeType":"text","value":"available here","marks":[],"data":{}}]},{"nodeType":"text","value":", for Platform license holders ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data Masking is enabled from within Conduktor Platform. You just need to click on the Data Masking option and then add the policies that you need. Please note that Data Masking is currently only available for Docker versions of the Platform, but will be coming soon to the web app.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Summary","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Security features are rarely a focus but are highly important. Conduktor Platform ensures that you don't ever have to worry about the security of your Kafka or your Kafka tools. Our security just works, no real setup needed. If you want to see all of these features in action, ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"start your free trial of Conduktor today","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"15jwOBqpxqSAOy2eOO4S0m","type":"Entry","createdAt":"2021-07-27T20:38:16.633Z","updatedAt":"2022-06-02T07:44:08.383Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":7,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Stéphane","surname":"Derosiaux","title":"Chief Technology Officer","twitter":"https://twitter.com/sderosiaux","github":"https://github.com/sderosiaux","linkedIn":"https://www.linkedin.com/in/st%C3%A9phane-derosiaux","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5fFFo1Rzqa8ZPG0VhWse8L","type":"Asset","createdAt":"2021-10-08T17:47:36.618Z","updatedAt":"2022-09-25T07:22:02.969Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":5,"locale":"en-US"},"fields":{"title":"Stephane Derosiaux","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5fFFo1Rzqa8ZPG0VhWse8L/ba8a3398e3326fc196029638925f94ea/stephane-d.png","details":{"size":49394,"image":{"width":192,"height":192}},"fileName":"stephane-d.png","contentType":"image/png"}}}}},"publishDate":"2022-11-17T14:00+00:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3E6ZUkOch78L2MHeicgr5h","type":"Entry","createdAt":"2022-09-22T17:37:39.887Z","updatedAt":"2022-09-25T13:56:42.846Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Introducing the Conduktor Platform","description":"We are delighted to announce the launch of the all-new Conduktor Platform, the next step in your data streaming journey","noIndex":false}},"slug":"introducing-the-conduktor-platform","title":"Introducing the Conduktor Platform","navbarDescription":"We are delighted to announce the launch of the all-new Conduktor Platform, the next step in your data streaming journey","description":"The Conduktor Platform is a new product that will transform your approach to Apache Kafka, providing solutions for Kafka management, testing, monitoring, data quality, security, and data governance.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"27BIbCbntoYoCbqMRXYW9b","type":"Asset","createdAt":"2022-09-22T17:37:23.459Z","updatedAt":"2022-09-25T13:58:01.012Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"platform-blog","description":"The Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/27BIbCbntoYoCbqMRXYW9b/f143238ccd63e2359af099956911d5e9/platform-blog.png","details":{"size":1877538,"image":{"width":2398,"height":1256}},"fileName":"platform-blog.png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"At Conduktor, our goal has always been to make ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/"},"content":[{"nodeType":"text","value":"Apache Kafka","marks":[],"data":{}}]},{"nodeType":"text","value":" simple for everyone. Since our founding several years ago, we have always aimed to deliver the best user interface tool available. Conduktor's original desktop incarnation is now used by over ","marks":[],"data":{}},{"nodeType":"text","value":"100'000 developers and 14'000 organizations","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". Now, we're ready to take the next step and transform the entire Kafka ecosystem, end-to-end.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Conduktor Platform will lead that transformation. Built on top of the UI foundations that you already know and love, the ","marks":[],"data":{}},{"nodeType":"text","value":"Conduktor Platform is available as both a SaaS web app and as a Docker application","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":", delivering one complete tool for Apache Kafka with many solutions.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The core Conduktor Console available in the Platform will have a significantly expanded free tier, with no time limit","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". You can also see it in action immediately with our demo environment, letting you play around with the Platform and with Kafka. Just head to ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"https://www.conduktor.io/get-started/","marks":[],"data":{}}]},{"nodeType":"text","value":" to check it out.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"The New Solutions of the Conduktor Platform","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The arrival of the new Conduktor Platform means new capabilities and features that will improve your experience with Apache Kafka. I made this 20 minute walkthrough for everything in the Platform, but if you prefer to read everything is below.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"The Conduktor Console","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Conduktor Console is a powerful UI for Apache Kafka, iterating over our previous product, Conduktor desktop. The Console equips Kafka users with much simpler and easier approaches to many tasks. We’ve divided the Console’s capabilities into three main areas:","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Explorer","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"First, there is the power of ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/explorer/"},"content":[{"nodeType":"text","value":"data exploration","marks":[],"data":{}}]},{"nodeType":"text","value":". Being able to clearly see which topics are being used and the status of your producers and consumers is vital. Newcomers to Kafka in particular can find it confusing. The Conduktor Console presents all of your topics, producers, consumers, and other Kafka essentials in an easy-to-understand, centralized, UI.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4FYbT19QBDBDHYblpQ8SZ5","type":"Asset","createdAt":"2022-09-23T08:55:17.826Z","updatedAt":"2022-09-25T13:56:52.209Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Data-explore-blog","description":"Data Exploration in Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4FYbT19QBDBDHYblpQ8SZ5/63d7bc1770177d9cb9e5997ef9c51255/blog_-_data-explorer.png","details":{"size":63003,"image":{"width":1200,"height":640}},"fileName":"blog - data-explorer.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Operations","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor does all the tricky stuff for you. You can get your standard ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/operations/"},"content":[{"nodeType":"text","value":"operations ","marks":[],"data":{}}]},{"nodeType":"text","value":"done with a few clicks. This provides a repeatable and standardized deployment and development experience. No need to memorize the more than 200 API calls for Apache Kafka.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7l1ObctMKuQCW0rHP89QAU","type":"Asset","createdAt":"2022-09-23T08:54:50.768Z","updatedAt":"2022-09-25T13:56:57.129Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"operations-blog","description":"Comparison of Schemas in Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7l1ObctMKuQCW0rHP89QAU/82a6f51f92e479686268311001095f8e/blog_-_operations.png","details":{"size":56212,"image":{"width":1200,"height":640}},"fileName":"blog - operations.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Troubleshooting","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Inevitably, you’ll run into some issues with Kafka. Big or small, problems need to be fixed, but Kafka is complex and difficult to navigate. ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/troubleshooting/"},"content":[{"nodeType":"text","value":"Conduktor troubleshooting ","marks":[],"data":{}}]},{"nodeType":"text","value":"surfaces errors for you rapidly, highlighting where the problem occurred and making it much easier to fix. And when you’ve already got the power of the Console’s Kafka operations, everything becomes much easier.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4aeThsBXw4sB2VxAODwBET","type":"Asset","createdAt":"2022-09-23T09:13:03.602Z","updatedAt":"2022-09-25T13:57:02.803Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"blog - troubleshooting","description":"Troubleshooting in the Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4aeThsBXw4sB2VxAODwBET/2cd0b3a608f7e076b3faa20cf0ef241d/blog_-_troubleshooting.png","details":{"size":75615,"image":{"width":1200,"height":640}},"fileName":"blog - troubleshooting.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conduktor Testing","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Testing Kafka applications has always been incredibly difficult to do. Not anymore. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/testing/"},"content":[{"nodeType":"text","value":"Conduktor Testing","marks":[],"data":{}}]},{"nodeType":"text","value":" can provide end-to-end testing in a visual interface that everyone can understand and collaborate on. Tests can be re-used again and again across teams, with every part of the tests shown.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"29afTe3OdmUn0bIlX1bPR8","type":"Asset","createdAt":"2022-09-23T08:20:05.110Z","updatedAt":"2022-09-25T13:57:08.465Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"testing-blog","description":"Conduktor Testing ","file":{"url":"//images.ctfassets.net/lm7q36eaz111/29afTe3OdmUn0bIlX1bPR8/7a2a6b553543b481dba458680a3fa81f/blog_-_testing.png","details":{"size":51581,"image":{"width":1200,"height":640}},"fileName":"blog - testing.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Topic Analyzer","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Topic Analyzer works for every part of your team, giving you a Kafka expert as a service.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Newcomers get information to help them learn faster. More experienced users can uncover hidden aspects of Kafka that they never thought about. And the experts get to see where they need to focus their time and which teams need their help the most.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3KBSCsKnHGp6gDFo54bSXR","type":"Asset","createdAt":"2022-09-23T10:09:47.881Z","updatedAt":"2022-09-25T13:57:12.920Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"blog - topic-analyzer","description":"Topic Analyzer in the Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3KBSCsKnHGp6gDFo54bSXR/d83bf126ab8616e67f0effccb8bce9b4/blog_-_topic-analyzer.png","details":{"size":58592,"image":{"width":1200,"height":640}},"fileName":"blog - topic-analyzer.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Chaos","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor has always been at the cutting edge of Apache Kafka, a relatively new and complicated technology. So, it was natural for the team to take on the challenge of Chaos Engineering, first conceptualized in 2011 and still rarely seen. Chaos Engineering attempts to simulate the chaos of the real world on applications, introducing failures and faults to see how your system holds up.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Chaos performs this task for your Kafka applications. Even if your development proceeds smoothly and tests go well, it's impossible to know if everything will hold up once it goes live, especially for Kafka which has its own unique foibles. Chaos gives you peace of mind for production, delivering more robust and resilient applications and letting you sleep soundly.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conduktor Monitoring","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Everyone understands the need for monitoring of applications, but there are few solutions designed for Kafka. Building effective Kafka monitoring into existing systems will take loads of resources, and in the end, you may not even be monitoring the really important metrics.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor provides an opinionated, best-practices based ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/monitoring/"},"content":[{"nodeType":"text","value":"monitoring ","marks":[],"data":{}}]},{"nodeType":"text","value":"solution that surfaces critical issues while downplaying false alerts. There is no complicated setup and it integrates easily into existing systems. It just works.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5kgOFCbnLYPfceiLsRCyZE","type":"Asset","createdAt":"2022-09-23T08:21:05.107Z","updatedAt":"2022-09-25T13:57:18.113Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"monitoring-blog","description":"Conduktor Monitoring","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5kgOFCbnLYPfceiLsRCyZE/daa8600920231b6a1673be6bcb6101ff/blog_-_monitoring.png","details":{"size":49985,"image":{"width":1200,"height":640}},"fileName":"blog - monitoring.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conduktor Alerting","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/alerting/"},"content":[{"nodeType":"text","value":"Alerting ","marks":[],"data":{}}]},{"nodeType":"text","value":"and Monitoring go hand-in-hand; you can’t have one without the other. So naturally, Conduktor offers both.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There were several challenges we sought to overcome with our alerts. First, our alerts make life easier for ops by making alerts simple and easy to understand. Second, our alerts make it clear which application is having trouble, showing you where problems occur and integrating with Slack. Real-time alerts that make sense are ready to go.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1aMNqNzDOSWI6yuZprYJlQ","type":"Asset","createdAt":"2022-09-23T08:22:48.137Z","updatedAt":"2022-09-25T13:57:22.864Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"alerting-blog","description":"Conduktor Alerting","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1aMNqNzDOSWI6yuZprYJlQ/1c99b8355b4f10e7fd44cbe07fd25db9/blog_-_monitoring-1.png","details":{"size":55577,"image":{"width":1200,"height":640}},"fileName":"blog - monitoring-1.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Topic As A Service","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Despite the complexity, Kafka is a great service and once someone starts using it, it will inevitably spread around a team or organization. This is fine at first, but once you come to work together things can get complicated. All those different conventions and protocols won’t fit without a lot of work, and soon enough you spend more time maintaining integration architecture than actually using Kafka.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/topic-service/"},"content":[{"nodeType":"text","value":"Topic As A Service ","marks":[],"data":{}}]},{"nodeType":"text","value":"lets you set the standard for how you work with Kafka for everyone, right from the start. Topic Creation becomes simple and even junior developers are able to work with Kafka without worry. Plus, topic actions are bound to identities, giving Ops a clear trail.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4ySMI8mmHRX19NinKoBTCb","type":"Asset","createdAt":"2022-09-23T08:47:15.237Z","updatedAt":"2022-09-25T13:57:28.495Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"topic-as-a-service-blog","description":"Topic as a Service","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4ySMI8mmHRX19NinKoBTCb/db6ece7c4f15efdb171dc5ae02d166e2/blog_-_taas.png","details":{"size":56724,"image":{"width":1200,"height":640}},"fileName":"blog - taas.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Data Masking","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There hasn't been a solution to make Kafka compliant with all the regulations out there, until now. We can’t make GDPR go away, but we can make sure that Personal Identifiable Information is protected and your team members only see what they are supposed to. ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/data-masking/"},"content":[{"nodeType":"text","value":"Data Masking ","marks":[],"data":{}}]},{"nodeType":"text","value":"hides sensitive data from the wrong people, keeping you in compliance and keeping your organization working on Kafka.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7MScEeJXQS1Lu3qA5FIYPr","type":"Asset","createdAt":"2022-09-23T10:11:08.868Z","updatedAt":"2022-09-25T13:57:32.763Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"blog - data-masking","description":"Data Masking in the Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7MScEeJXQS1Lu3qA5FIYPr/9c0557f8753d2dfdcb114df358c10f40/blog_-_data-masking.png","details":{"size":56855,"image":{"width":1200,"height":640}},"fileName":"blog - data-masking.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"RBAC and SSO","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Security is essential to your company. The Conduktor Platform offers ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/rbac/"},"content":[{"nodeType":"text","value":"Role-Based Access Control ","marks":[],"data":{}}]},{"nodeType":"text","value":"at the topic level, giving more granular control than any other solution. Together with ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/sso/"},"content":[{"nodeType":"text","value":"Single Sign-On ","marks":[],"data":{}}]},{"nodeType":"text","value":"that supports LDAP and OpenID, it’s easy to meet the security requirements for your organization.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1YxYLexJyxZ1LIeSibYEfb","type":"Asset","createdAt":"2022-09-23T08:51:37.864Z","updatedAt":"2022-09-25T13:57:41.743Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"rbac-blog","description":"RBAC and SSO in Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1YxYLexJyxZ1LIeSibYEfb/fc3395d2d3fcf387536b8d8bbf98078e/blog_-_rbac___sso.png","details":{"size":55655,"image":{"width":1200,"height":640}},"fileName":"blog - rbac + sso.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conduktor Audit","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The majority of modern enterprises use Kafka for their infrastructure, making it important to keep track of changes. Unfortunately, Kafka remains something of a black box, preventing your Ops from understanding everything that occurs.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/audit/"},"content":[{"nodeType":"text","value":"Conduktor Audit ","marks":[],"data":{}}]},{"nodeType":"text","value":"shows who exactly has access to what, who has done what, and when they did it. It works across clusters and fits seamlessly into the rest of the Conduktor Platform.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3414hEOO0A1i9k0ihpldAZ","type":"Asset","createdAt":"2022-09-23T09:14:11.852Z","updatedAt":"2022-09-25T13:57:37.296Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"blog - audit","description":"Conduktor Audit Logs","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3414hEOO0A1i9k0ihpldAZ/c835db0005214eccfb2850edf0c9fed3/blog_-_audit.png","details":{"size":63288,"image":{"width":1200,"height":640}},"fileName":"blog - audit.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Summary","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We are delighted to release the Conduktor Platform for our users and help make every part of Apache Kafka simple. Everything is ready to go, all you need to do is sign up and either run the Docker command or get started online. This is just the beginning of your data streaming journey.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For our Conduktor Desktop users, we will continue to offer this service and provide support where needed, but switching to the Platform will not entail any change to your current ways of working and this is where the most exciting features will be. ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Why not give it a try?","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2SX6zpOndgjWDVop0j59Uz","type":"Entry","createdAt":"2021-10-08T17:41:29.513Z","updatedAt":"2022-09-27T10:40:02.619Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":6,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Stéphane","surname":"Maarek","title":"Chief Evangelist","twitter":"https://twitter.com/StephaneMaarek","github":"https://github.com/simplesteph","linkedIn":"https://www.linkedin.com/in/stephanemaarek/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7y6tdCR7qU2yEWMJmQDS23","type":"Asset","createdAt":"2021-10-08T17:40:54.512Z","updatedAt":"2022-09-25T07:11:34.678Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Stephane Maarek","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7y6tdCR7qU2yEWMJmQDS23/523069a652132097fc9825d7831c0790/stephane-m.png","details":{"size":47223,"image":{"width":192,"height":192}},"fileName":"stephane-m.png","contentType":"image/png"}}}}},"publishDate":"2022-09-27T11:00+01:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"DYkArPvigGFkoMEnPS0Ym","type":"Entry","createdAt":"2022-06-24T13:19:09.349Z","updatedAt":"2022-09-25T13:38:26.195Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Validating Event-Driven Architectures with Conduktor Testing","description":"We demonstrate how to use our latest product to validate workflows and test data integrity at each stage of your system architecture.","noIndex":false}},"slug":"validating-event-driven-architectures-with-conduktor-testing","title":"Validating Event-Driven Architectures","navbarDescription":"We demonstrate how to use our latest product to validate workflows and test data integrity.","description":"Demonstrating how to use our latest product to validate workflows and test data integrity at each stage of your system architecture.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"J6ZtDosemzvkWs7lboMxy","type":"Asset","createdAt":"2022-06-24T14:03:25.003Z","updatedAt":"2022-09-25T18:37:23.307Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":5,"locale":"en-US"},"fields":{"title":"Event Driven Testing Image","description":"Validating Event-Driven Architectures with Conduktor Testing","file":{"url":"//images.ctfassets.net/lm7q36eaz111/J6ZtDosemzvkWs7lboMxy/e79774d98a949f91acb772128e2a661c/blog_validating_event_driven_architectures__1_.png","details":{"size":102648,"image":{"width":1200,"height":630}},"fileName":"blog_validating_event_driven_architectures (1).png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Event-driven architectures have become common ground for developing modern applications. Their loosely coupled, low-latency, and scalable nature provide organizations with the agility required to be competitive in any market. Up until now, navigating how to test them was the less glamorous side of real-time decision making and reactive applications. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/testing/"},"content":[{"nodeType":"text","value":"Conduktor Testing","marks":[],"data":{}}]},{"nodeType":"text","value":", you can empower QAs with limited Kafka expertise, write end-to-end tests quickly, and automate their execution in ","marks":[],"data":{}},{"nodeType":"text","value":"any","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" CI/CD environment. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ultimately, we’ll help you focus on ","marks":[],"data":{}},{"nodeType":"text","value":"what to test","marks":[{"type":"underline"}],"data":{}},{"nodeType":"text","value":" (business scenarios that matter), not ","marks":[],"data":{}},{"nodeType":"text","value":"how to test it","marks":[{"type":"underline"}],"data":{}},{"nodeType":"text","value":" (implementing an async testing framework).","marks":[],"data":{}}]},{"nodeType":"hr","data":{},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Case Study | Event-Driven Retail System","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In this post, we will demonstrate how to compose an end-to-end Test Scenario for an event-driven system composed of multiple topics. This tutorial will demonstrate:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Testing the ","marks":[],"data":{}},{"nodeType":"text","value":"workflow","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" to validate the integrations","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Are messages correctly broadcast to Kafka topics from the external applications?","marks":[{"type":"italic"}],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Does the workflow complete within an agreed SLA?","marks":[{"type":"italic"}],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Testing the ","marks":[],"data":{}},{"nodeType":"text","value":"data correctness","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" at each step:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Was our enrichment successful, based on the userId present in the message?","marks":[{"type":"italic"}],"data":{}}]}]}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The pre-requisite to creating tests are ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/platform/testing/getting-started/connect-to-a-kafka-cluster/"},"content":[{"nodeType":"text","value":"cluster configurations","marks":[],"data":{}}]},{"nodeType":"text","value":", which are defined in your workspace during setup. This involves specifying the bootstrap servers, schema registry URL (if using it), and any additional broker configurations. The Conduktor ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/platform/testing/getting-started/install-the-testing-agent/"},"content":[{"nodeType":"text","value":"Testing Agent","marks":[],"data":{}}]},{"nodeType":"text","value":" allows you to reach any public, private or local cluster that your host has access to.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"wvKqiaKezIsW61CnVwdPf","type":"Asset","createdAt":"2022-06-24T13:23:15.631Z","updatedAt":"2022-09-25T07:14:04.790Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"TestingBlog1","description":"Conduktor Testing UI","file":{"url":"//images.ctfassets.net/lm7q36eaz111/wvKqiaKezIsW61CnVwdPf/c12e46347691370e96fdf5191b8f51ee/Image1_-_Clusters__1_.png","details":{"size":207947,"image":{"width":3680,"height":2268}},"fileName":"Image1 - Clusters (1).png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Architecture Reference","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For the purpose of this tutorial, we will demonstrate a retail system consisting of multiple stream processing steps. The architecture and message flows are demonstrated below.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2Jg5mTEVNR5hrnWKJlSFOX","type":"Asset","createdAt":"2022-06-24T13:24:09.582Z","updatedAt":"2022-09-25T07:14:09.021Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"TestingBlog2","description":"Architecture and Message Flows","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2Jg5mTEVNR5hrnWKJlSFOX/60c89ad133205b1b82b566ca5ea9f8a0/Image2_-_ArchRef.png","details":{"size":56911,"image":{"width":2694,"height":1006}},"fileName":"Image2 - ArchRef.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The event flow can be described as:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Message is produced into the ","marks":[],"data":{}},{"nodeType":"text","value":"pageviews","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" topic","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"User service consumes the message => enriches it with additional data => then produces an enriched message into the ","marks":[],"data":{}},{"nodeType":"text","value":"pageviews_enriched","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" topic","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The promotions service consumes the enriched message and => assuming certain business rules are met => produces another message into the ","marks":[],"data":{}},{"nodeType":"text","value":"upsell_events","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" topic","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Visually representing the Test Scenario","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are 3 Kafka topics involved in the system architecture. This means there are multiple stages at which it’s possible to check the data and validate if the intermediary applications ","marks":[],"data":{}},{"nodeType":"text","value":"(user service, promotions service) ","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"have fulfilled their responsibilities. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To build the scenario in Conduktor Testing, we use nodes on a canvas to represent tasks relating to Kafka. An example task could be producing or consuming from a Kafka topic. We’re also working to add support for other protocols such as HTTP.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4AhraM0FrU7D6EmvGT62Pe","type":"Asset","createdAt":"2022-06-24T13:25:33.502Z","updatedAt":"2022-09-25T07:14:14.333Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"TestingBlog3","description":"Visual Representation of a Test","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4AhraM0FrU7D6EmvGT62Pe/d09db8f5662fc8f66e1d9562335e92fd/Image3_-_Overview.png","details":{"size":235062,"image":{"width":3680,"height":2268}},"fileName":"Image3 - Overview.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"It’s worth noting that the external applications ","marks":[],"data":{}},{"nodeType":"text","value":"(user service, promotions service)","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" are not represented on the graph. Though, they must be available for a valid test execution as they’re responsible for intermediate processing steps. In each case, the services consume a message and do ","marks":[],"data":{}},{"nodeType":"text","value":"something","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":", before publishing a new message into another Kafka topic. It’s the resultant output in Kafka that we are testing at each stage.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"KsqqLGbu4aT8Zbdt836gT","type":"Asset","createdAt":"2022-06-24T13:26:38.055Z","updatedAt":"2022-09-25T07:14:18.982Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"TestingBlog4","description":"Annotated Test Visual","file":{"url":"//images.ctfassets.net/lm7q36eaz111/KsqqLGbu4aT8Zbdt836gT/6b57703620b391262e57bc327ad34c37/Image4_-_Annotated.png","details":{"size":201878,"image":{"width":2372,"height":1414}},"fileName":"Image4 - Annotated.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Breaking it down","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Producer Task","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The first node on the canvas is a producer task. It’s responsible for publishing a message into the ","marks":[],"data":{}},{"nodeType":"text","value":"pageviews","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" topic. The relevant cluster and topic are configured on the ","marks":[],"data":{}},{"nodeType":"text","value":"General","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" tab. Then, a JSON value is set as the message value on the ","marks":[],"data":{}},{"nodeType":"text","value":"Data","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" tab. You can also configure ","marks":[],"data":{}},{"nodeType":"text","value":"randomly generated","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" data for more dynamic test conditions. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The user service will take care of enrichment based on the ","marks":[],"data":{}},{"nodeType":"text","value":"userId","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" presented in this message. It’s possible to check the ","marks":[],"data":{}},{"nodeType":"text","value":"RecordMetadata","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" (offset, partition, topic, etc.) associated with a published message, but in this case we will not configure a producer check.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4mSqyQsdZEXpLmUSgFpfDj","type":"Asset","createdAt":"2022-06-24T13:27:59.647Z","updatedAt":"2022-09-25T07:14:23.692Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"TestingBlog5","description":"Producer Task","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4mSqyQsdZEXpLmUSgFpfDj/2c3380806a08bdae2495af96907c67d6/Image5_-_Producer.png","details":{"size":271127,"image":{"width":3680,"height":2268}},"fileName":"Image5 - Producer.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Consumer Task - Check Enriched","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This task is used to consume the enriched message that we expect to be emitted from the user service (assuming it’s working correctly). ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In the ","marks":[],"data":{}},{"nodeType":"text","value":"Data","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" tab, the deserialization format for consuming the enriched message is set. In this case, we expect a JSON value. It’s also necessary to configure lifecycle rules that determine when the task should stop consuming, or when it should be considered to have failed.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"11UTkJrLWgEshgDK5Cby2S","type":"Asset","createdAt":"2022-06-24T13:28:41.932Z","updatedAt":"2022-09-25T07:14:28.930Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"TestingBlog6","description":"Data tab","file":{"url":"//images.ctfassets.net/lm7q36eaz111/11UTkJrLWgEshgDK5Cby2S/ba5aeef2f7b8eddcfe5782becec07c4e/Image6_-_Consumer_Enriched.png","details":{"size":280826,"image":{"width":3216,"height":1982}},"fileName":"Image6 - Consumer Enriched.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In this case, we expect 1 enriched message for each message produced. Therefore, the stop condition is set to 1 record. If messages were continuously being produced into the topic, you could use a filter to intercept the correct one.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The final step relating to this task is to create a check on the consumed message. Below demonstrates the expected format of the message post-enrichment.","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3v1imqPA4olAhOg6LNh4Ca","type":"Entry","createdAt":"2023-01-18T17:45:27.548Z","updatedAt":"2023-01-18T17:45:27.548Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"json","code":"{\n \"timestamp\":1652012999698,\n \"userId\":12345,\n \"pageId\":200,\n \"pageName\":\"Checkout\",\n \"country\":\"US\",\n \"planType\":\"Free\",\n \"interests\":\"Gaming\"\n}"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To test data inside the message, it’s possible to use ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://stedolan.github.io/jq/"},"content":[{"nodeType":"text","value":"jq","marks":[],"data":{}}]},{"nodeType":"text","value":" to access the attributes for the purpose of an expression. In this case, I will validate that the ","marks":[],"data":{}},{"nodeType":"text","value":"planType","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" relevant to this user was correctly enriched by the user service. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Below demonstrates an expression that asserts the ","marks":[],"data":{}},{"nodeType":"text","value":"planType ","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"for this user to be ","marks":[],"data":{}},{"nodeType":"text","value":"Free","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":". If any other string value is received, this test would be considered a failure. ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4mhhWKGUeGTH6un1mDBURt","type":"Asset","createdAt":"2022-06-24T13:30:08.904Z","updatedAt":"2022-09-25T07:14:33.442Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"TestingBlog7","description":"planType expression","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4mhhWKGUeGTH6un1mDBURt/7d9de364a13f5029530a9a7534c135d2/Image7_-_jq_Check.png","details":{"size":255835,"image":{"width":3680,"height":2268}},"fileName":"Image7 - jq Check.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Consumer Task - Check Upsell","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The final task is used to ensure that the business logic for the promotions service is working correctly. The business rule assumes that ","marks":[],"data":{}},{"nodeType":"text","value":"Free","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" members who visit the checkout page should be upsold promotional products. The expectation is a message produced into the ","marks":[],"data":{}},{"nodeType":"text","value":"upsell_events","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" topic.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"From a business perspective, it’s important this happens within an agreed SLA. For the purpose of this example, let’s assume that the ","marks":[],"data":{}},{"nodeType":"text","value":"SLA","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" is set at ","marks":[],"data":{}},{"nodeType":"text","value":"500ms","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This can be addressed via the lifecycle rules configured on the ","marks":[],"data":{}},{"nodeType":"text","value":"Data","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" tab. The fail condition is set to an elapsed time of ","marks":[],"data":{}},{"nodeType":"text","value":"500ms. ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"Meaning, the test will be considered a failure if no message is received within that interval.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3Xvx2Zwsjiv6anIENKxCjj","type":"Asset","createdAt":"2022-06-24T13:54:46.634Z","updatedAt":"2022-09-25T07:14:37.991Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"TestingBlog8","description":"Fail conditions","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3Xvx2Zwsjiv6anIENKxCjj/518e4ee9d4b813b225ef3f14bd541a9c/Image8_-_CheckUpsell.png","details":{"size":259675,"image":{"width":3680,"height":2268}},"fileName":"Image8 - CheckUpsell.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Test Execution","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Now that the workflow and checks have been configured, the only remaining step is to execute the scenario and observe the system behavior. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Manually execute the test scenario using the","marks":[],"data":{}},{"nodeType":"text","value":" Run","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" button.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6paaOPkkuMApbhCzrKau8a","type":"Asset","createdAt":"2022-06-24T13:55:25.623Z","updatedAt":"2022-09-25T07:14:42.312Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"TestingBlog9","description":"Test Execution","file":{"url":"//images.ctfassets.net/lm7q36eaz111/6paaOPkkuMApbhCzrKau8a/59f4292d69e3d7e60891dba87042a602/Image9_-_Test_Execution.png","details":{"size":292712,"image":{"width":3680,"height":2268}},"fileName":"Image9 - Test Execution.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In the above case, we can see that the first 2 tasks passed. However, the end-to-end test scenario has ","marks":[],"data":{}},{"nodeType":"text","value":"failed ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"due to the final task. Hovering over the execution events will provide more detail on the reasons why. ","marks":[],"data":{}}]},{"nodeType":"blockquote","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Consumed reached the Timeout fail condition","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This mean the business SLA of 500ms, associated with the final task, was not met","marks":[],"data":{}}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You can also preview the data associated with messages produced and consumed events. Below demonstrates previewing of the ‘Record consumed’ event, associated with the ‘Check Enriched’ task.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4QdEh7PkSPL5zTj20r4tF2","type":"Asset","createdAt":"2022-06-24T13:57:28.062Z","updatedAt":"2022-09-25T07:14:46.691Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"TestingBlog10","description":"Message Preview","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4QdEh7PkSPL5zTj20r4tF2/27a694a2b0b9cdc2f69fec9d285d1928/Image10_-_Preview_Record.png","details":{"size":275558,"image":{"width":3680,"height":2268}},"fileName":"Image10 - Preview Record.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Navigating to the ","marks":[],"data":{}},{"nodeType":"text","value":"Checks ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"tab will detail the result of any test checks. In this case, the check on the enriched message was successful. Meaning, the user was correctly attributed to a ","marks":[],"data":{}},{"nodeType":"text","value":"Free","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" plan by the user service. ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1XtcURjY98rWTsrZhXc9ue","type":"Asset","createdAt":"2022-06-24T13:58:12.627Z","updatedAt":"2022-09-25T07:14:51.712Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"TestingBlog11","description":"Checks Tab","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1XtcURjY98rWTsrZhXc9ue/2c6a44c1cac52e9d834e8f430e0f1a24/Image11_-_Check_Success.png","details":{"size":252835,"image":{"width":3680,"height":2268}},"fileName":"Image11 - Check Success.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Great! To recap, we have proven how to:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Visually architect a business scenario we want to test ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Test the business data that are output into Kafka from the user service","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Test whether the workflow executes within an agreed business SLA","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Of course, everything that’s been demonstrated in this blog post has involved the UI and manual test executions. To take this further and enable continuous, automated testing, it’s possible to export the test configuration for integration in any CI/CD environment. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You can see an example Github action ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/platform/testing/features/ci-cd-automation#create-a-run-configuration"},"content":[{"nodeType":"text","value":"here","marks":[],"data":{}}]},{"nodeType":"text","value":", but we’ll save a deeper dive on that for a future post!","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Sign up","marks":[],"data":{}}]},{"nodeType":"text","value":" and start using Conduktor Testing today! Cheers!\n","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1JmKff2d8J5C93f4yUSfCV","type":"Entry","createdAt":"2021-12-08T13:19:31.889Z","updatedAt":"2021-12-08T13:19:31.889Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"James","title":"Technical Product Manager","linkedIn":"https://www.linkedin.com/in/james-white-301b5550/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"65GcHlPUFtafUImTScSJjv","type":"Asset","createdAt":"2021-12-08T13:18:31.985Z","updatedAt":"2022-09-25T14:00:19.941Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"james","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/65GcHlPUFtafUImTScSJjv/d3f2ce92a6a6e7c8b4abc37b3da6fafa/james.png","details":{"size":47354,"image":{"width":192,"height":192}},"fileName":"james.png","contentType":"image/png"}}}}},"publishDate":"2022-06-24T18:00+01:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"55JoqhwlSqlQjeebDkvmlH","type":"Entry","createdAt":"2022-03-22T14:52:12.177Z","updatedAt":"2022-09-25T13:51:57.768Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":6,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Kafka vs Spark - An Introduction","description":"This article introduces two Apache projects that have been popular in the big data processing space, Apache Kafka and Apache Spark. ","noIndex":false}},"slug":"comparing-apache-kafka-vs-apache-spark","title":"Kafka & Spark - A match made in data","navbarDescription":"A look at how Apache Kafka & Apache Spark perfectly complement each other to solve for many use cases.","description":"A look at how Apache Kafka & Apache Spark perfectly complement each other to solve for many use cases.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"uLiaEWErJs1bM7wyvUkGb","type":"Asset","createdAt":"2022-03-24T07:46:11.178Z","updatedAt":"2022-09-25T08:16:34.980Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Apache Kafka Spark Use Cases","description":"Kafka vs Spark blog image","file":{"url":"//images.ctfassets.net/lm7q36eaz111/uLiaEWErJs1bM7wyvUkGb/05ad3cd0fd19ba232200cb1cc69c10de/Apache_Kafka_vs_Apache_Spark.jpg","details":{"size":39724,"image":{"width":640,"height":427}},"fileName":"Apache Kafka vs Apache Spark.jpg","contentType":"image/jpeg"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As we see in many organizations today, deriving meaningful insights from raw data is still a challenging task. Having the right data processing technology stack at your disposal will significantly simplify the data ingestion, storage, and processing tasks at scale, enabling you to make timely data-driven business decisions.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are a variety of big data technologies on the market, including Apache Kafka, Apache Spark, Flink, Apache Storm, and others. This article will focus on two prominent technologies, ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/"},"content":[{"nodeType":"text","value":"Apache Kafka","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" and ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://spark.apache.org/"},"content":[{"nodeType":"text","value":"Apache Spark","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", their architecture, and how they complement each other.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is Apache Kafka","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka is a distributed event streaming platform, which can ingest events from different source systems at scale and store them in a fault-tolerant distributed system called a ","marks":[],"data":{}},{"nodeType":"text","value":"Kafka cluster","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". A Kafka cluster is a collection of ","marks":[],"data":{}},{"nodeType":"text","value":"brokers","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" who organize events into ","marks":[],"data":{}},{"nodeType":"text","value":"topics","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" and store them durably for a configurable amount of time. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka provides APIs for reading and writing data. Applications such as web applications, IoT devices, and Microservices could use the ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/complete-kafka-producer-with-java/"},"content":[{"nodeType":"text","value":"Producer API","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" to write events into a Kafka topic. Consumer applications like stream processors and analytics databases subscribe to a topic and read events using ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/complete-kafka-consumer-with-java/"},"content":[{"nodeType":"text","value":"Consumer API","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":". Multiple producers can simultaneously produce events to a topic, which leads to virtually unlimited scalability.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"mSYc3d9f4n5lKQ8W4GC0g","type":"Asset","createdAt":"2022-02-08T12:10:55.338Z","updatedAt":"2022-09-25T08:15:59.579Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Apache Kafka Architecture","description":"Diagram showing the basic architecture of Apache Kafka vs Amazon Kinesis.","file":{"url":"//images.ctfassets.net/lm7q36eaz111/mSYc3d9f4n5lKQ8W4GC0g/d01b03f76f0f113896ece90b490bba83/Apache_Kafka_Architecture.png","details":{"size":89744,"image":{"width":2880,"height":2012}},"fileName":"Apache Kafka Architecture.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is Apache Spark","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Spark is an open-source, distributed computing framework used for resilient data processing on large-scale data sets. Spark is much faster than its predecessor, Hadoop, which works based on the ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://en.wikipedia.org/wiki/MapReduce"},"content":[{"nodeType":"text","value":"MapReduce","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" programming model. The secret for being faster is that Spark runs on memory (RAM), making it ideal for analyzing both batch and streaming datasets. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Spark has been designed around the concept of a ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://spark.apache.org/docs/latest/rdd-programming-guide.html#resilient-distributed-datasets-rdds"},"content":[{"nodeType":"text","value":"resilient distributed dataset (RDD)","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", which is a fault-tolerant collection of elements that can be operated on in parallel. Developers write Spark applications in any ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://spark.apache.org/docs/2.2.0/rdd-programming-guide.html"},"content":[{"nodeType":"text","value":"supported programming language","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" and deploy them on a Spark cluster to get a particular data processing job done. ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"EBXsz1As7EIXipVzxPvkj","type":"Asset","createdAt":"2022-03-22T15:13:48.749Z","updatedAt":"2022-09-25T08:16:04.885Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Figure 02 - Apache Spark Execution Model","description":"Diagram showing Apache Spark Execution Model","file":{"url":"//images.ctfassets.net/lm7q36eaz111/EBXsz1As7EIXipVzxPvkj/87f52bad81c15e3b63480aff306c160d/Apache_Spark_Execution_Model__1_.png","details":{"size":68968,"image":{"width":2880,"height":1428}},"fileName":"Apache Spark Execution Model (1).png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is Spark Structured Streaming","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Before exploring Spark Structured Streaming, it is worth distinguishing between bounded and unbounded data.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Bounded vs. unbounded data","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Bounded data is a finite dataset with a well-defined boundary, a start, and an end. Processing bounded data is relatively easy as it is a static dataset. Therefore, we can process them in batches. Bounded data is often called ","marks":[],"data":{}},{"nodeType":"text","value":"batch data","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". An example would be the customer orders collected over an hour, day, or month.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Unlike its “bounded” counterpart — batch data, unbounded data has no boundaries defined in terms of time. The data may have been arriving from the past, continuing today, and is expected to arrive in the future. Unbounded data is often called ","marks":[],"data":{}},{"nodeType":"text","value":"streaming data","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". An example would be clickstream events coming from a website; the data will continue to arrive as long as users browse.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7tohJ4mrucrtxYhgcd1uwd","type":"Asset","createdAt":"2022-03-22T15:14:51.619Z","updatedAt":"2022-09-25T08:16:09.683Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Figure 03- Bounded vs. Unbounded","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7tohJ4mrucrtxYhgcd1uwd/337678aa7ca5f37de0ccde0681705380/Bounded_vs._Unbounded_Data.png","details":{"size":86275,"image":{"width":3416,"height":1872}},"fileName":"Bounded vs. Unbounded Data.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In reality, a lot of data is unbounded because it arrives gradually over time: your users produced data yesterday and today, and they will continue to produce more data tomorrow. Unless you go out of business, this process never ends, so the dataset is never “complete” in any meaningful way. --— Martin Kleppmann, Designing Data-Intensive Applications","marks":[{"type":"italic"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Processing bounded and unbounded data requires different architectural paradigms, practices, and technologies--causing organizations to maintain two technology stacks.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Structured streaming: unified APIs for processing batch and streaming data","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html#overview"},"content":[{"nodeType":"text","value":"Structured Streaming","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" is a scalable and fault-tolerant streaming processing framework built on the Spark SQL engine. It lets you express computation on streaming data in the same way you express a batch computation on static data. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Structured Streaming is exposed to the developer through ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://spark.apache.org/docs/latest/sql-programming-guide.html"},"content":[{"nodeType":"text","value":"Dataset/DataFrame API","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" in Scala, Java, Python, and R. With the DataFrame API, developers just describe the query they want to run, input, and output locations, and optionally a few more details. The underlying Spark SQL engine then runs its query incrementally, maintaining enough state to recover from failure, and continuously updates the result as streaming data arrives. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There's a significant difference between Structured Streaming and its predecessor, ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://spark.apache.org/docs/latest/streaming-programming-guide.html"},"content":[{"nodeType":"text","value":"Spark Streaming","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", a micro-batch processing engine, which processes data streams as a series of small batch jobs. Since Spark 2.3, Structured Streaming supports a new low-latency processing mode called ","marks":[],"data":{}},{"nodeType":"text","value":"Continuous Processing","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":", which can achieve end-to-end latencies as low as one millisecond with at-least-once guarantees.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"How Kafka and Spark work together","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4IxRZZsq61FoUmkdCuCzH6","type":"Asset","createdAt":"2022-03-22T15:16:12.946Z","updatedAt":"2022-09-25T08:16:15.262Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Figure 04- Kafka feeding Spark to accomplish many use cases","description":"Diagram showing how Kafka annd Spark can work together to support many business objectives.","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4IxRZZsq61FoUmkdCuCzH6/562c9a316c354628d1fa352d31c23de2/Kafka_feeding_Spark_to_accomplish_many_use_cases.png","details":{"size":277401,"image":{"width":3440,"height":1476}},"fileName":"Kafka feeding Spark to accomplish many use cases.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka and Spark are entirely different technologies in their architecture and the use cases served. Despite being open-source and running on the JVM, Kafka and Spark have a particular thing in between--","marks":[],"data":{}},{"nodeType":"text","value":"they complement each other very well. ","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka provides durable storage for streaming data, whereas Spark reads and writes data to Kafka in a scalable and fault-tolerant manner. When combined, these technologies can be utilized to build large-scale data processing use cases. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This integration is made possible via Structured Streaming. For better understanding, let’s take a step back and dive deep into ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://spark.apache.org/docs/latest/sql-programming-guide.html#datasets-and-dataframes"},"content":[{"nodeType":"text","value":"DataFrames","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", a fundamental data structure you can find in Structured Streaming.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Understanding DataFrames API","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Developers access Structured Streaming functionalities through DataFrames. A DataFrame is a distributed collection of data organized into named columns in Spark. It is inspired by the data frames in R and Python (Pandas) and conceptually equivalent to a table in a relational database. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"DataFrame can read from and write data to various locations over popular file formats such as JSON files, Parquet files, and Hive tables. These locations include local file systems, distributed file systems (HDFS), cloud storage (S3), and external relational database systems via JDBC. In addition, through Spark SQL’s external data sources API, DataFrames can be extended to support any third-party data formats or sources. Existing third-party extensions already include Avro, CSV, ElasticSearch, and Cassandra.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Constructing a DataFrames requires only a few lines of code. The following is a Python example that reads a JSON file from an S3 bucket. A similar API is available in Scala and Java.","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7LrAcP3Kt4uv7V5Ujnf6at","type":"Entry","createdAt":"2023-01-18T18:04:59.039Z","updatedAt":"2023-01-18T18:04:59.039Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"java","code":"users = context.load(\"s3n://path/to/data.json\", \"json\")\nOnce constructed, DataFrames can be used for distributed data manipulation.\n# Create a new DataFrame that contains “young users” only young = users.filter(users.age < 21)\n# Alternatively, using Pandas-like syntax young = users[users.age < 21]\n# Increment everybody’s age by 1 young.select(young.name, young.age + 1)\n# Count the number of young users by gender young.groupBy(\"gender\").count()\n# Join young users with another DataFrame called logs young.join(logs, logs.userId == users.userId, \"left_outer\")\n"}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Apache Kafka support in Structured Streaming","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"DataFrames can read from and write data to Kafka like other data sources. Structured Streaming ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://spark.apache.org/docs/latest/structured-streaming-kafka-integration.html"},"content":[{"nodeType":"text","value":"integration with Kafka","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" (version 0.10 and higher) provides a unified batch and streaming API to view data published to Kafka as a DataFrame. When processing unbounded data in a streaming fashion, we use the same API to get the same data consistency guarantees as in batch processing.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka integration helps Spark to deliver three data processing styles.","marks":[],"data":{}}]},{"nodeType":"ordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Process streaming data","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Process batch data","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Replace Kafka Connect","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Stream processing on Kafka data","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka integration enables building stream processing applications with Structured Streaming. Spark engine constructs a DataFrame from a Kafka topic using the ","marks":[],"data":{}},{"nodeType":"text","value":"spark.readStream.format(“kafka”) ","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"method. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The following is a Scala example reading data from a Kafka topic, ","marks":[],"data":{}},{"nodeType":"text","value":"\"foo\".","marks":[{"type":"italic"}],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6QpLqMiNQM2nZutztLSpGn","type":"Entry","createdAt":"2023-01-18T18:05:28.928Z","updatedAt":"2023-01-18T18:05:28.928Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"java","code":"val df = spark.readStream.format(\"kafka\")\n .option(\"kafka.bootstrap.servers\", \"localhost:9092\")\n .option(\"subscribe\", \"foo\")\n .option(\"startingOffsets\", \"earliest\")\n .load()"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The DataFrame can then be used for further data manipulation as usual. Assuming that the foo topic consists of a list of words, we can calculate the frequency of each word and write it to a file as follows.","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4WIhu18KYO9bh2eRrrfgTw","type":"Entry","createdAt":"2023-01-18T18:06:21.117Z","updatedAt":"2023-01-18T18:06:21.117Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"java","code":"df.select(explode(split($ \"value\".cast(\"string\"), \"\\\\s+\")).as(\"word\"))\n .groupBy($ \"word\")\n .count)\n.writeStream\n .format(\"parquet\")\n .option(\"path\", \"/example/wordCount\")\n .option(\"checkpointLocation\", \"/streamcheckpoint\")\n .start.awaitTermination(30000)"}}},"content":[]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3avotUW5sJE8DrTUl5H0hZ","type":"Asset","createdAt":"2022-03-22T15:31:25.182Z","updatedAt":"2022-09-25T08:16:21.793Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Figure 05- Kafka Integration","description":"Apache Kafka Spark Integration Diagram ","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3avotUW5sJE8DrTUl5H0hZ/32b7dd90a7566036d40c4ee18beec7fe/Apache_Spark_Kafka_Integration.png","details":{"size":129450,"image":{"width":2880,"height":1428}},"fileName":"Apache Spark Kafka Integration.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Stream processing applications have two variations in their processing style as follows.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Event-at-a-time processing","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" - One event is processed at a time. For example, an event payload is evaluated for a pattern. If found, it is routed to a different system, otherwise dropped. That is also called stateless event processing.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Event stream processing","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" - A series of aggregate functions are applied to a stream of events aggregated over a time window. For example, consider calculating the average temperature of temperature sensor readings over the last minute. That is called stateful event processing.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Structured Streaming supports building both stateless and stateful applications. One great aspect it offers is that the developer is free from underlying streaming infrastructure. Once the application is deployed, the underlying Spark SQL engine ensures end-to-end exactly-once fault-tolerance guarantees so that developers do not have to reason about low-level aspects of streaming.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In Kafka’s ecosystem, this behavior is similar to what is offered by ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/31/documentation/streams/"},"content":[{"nodeType":"text","value":"Kafka Streams","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" and ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://ksqldb.io/"},"content":[{"nodeType":"text","value":"ksqlDB","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", two popular technologies for building stream processing applications on Kafka data. This means Spark Streaming can replace existing Kafka Streams/ksqlDB applications at some point. ","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Batch processing on Kafka data","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We can also create a DataFrame from a Kafka topic to process data in batch mode. That is useful for scenarios when historical data stored in a compacted Kafka topic is processed for backfilling or training ML models.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Like streaming mode, a DataFrame can be constructed from a Kafka topic, but we use ","marks":[],"data":{}},{"nodeType":"text","value":"spark.read.format(\"kafka\")","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" instead of","marks":[],"data":{}},{"nodeType":"text","value":" readStream.","marks":[{"type":"italic"}],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6nojJ7lU8ZTGbiOGTsFrsH","type":"Entry","createdAt":"2023-01-18T18:06:57.890Z","updatedAt":"2023-01-18T18:06:57.890Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"java","code":"val df = spark.readStream.format(\"kafka\")\n .option(\"kafka.bootstrap.servers\", \"localhost:9092\")\n .option(\"subscribe\", \"foo\")\n .option(\"startingOffsets\", \"earliest\")\n .load()"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Once constructed, the DataFrame can be used for further data manipulation.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"DataFrames as a replacement for Kafka Connect","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/documentation/#connect"},"content":[{"nodeType":"text","value":"Kafka Connect","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" allows Kafka to connect the external source and target systems for reading and writing data. A connection is established through a Connector, a reusable library that embeds the logic for connecting to a particular system and performing IO. S3, Elasticsearch, HDFS, and JDBC are some notable connectors in the Kafka ecosystem that enables the integration of different enterprise systems through Kafka.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We can achieve the same with Structured Streaming through DataFrames. As we saw above, DataFrames’ ability to read from and write to different data sources enables applications to easily combine data from disparate sources. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Developers can use DataFrames to read from one data source, process it in the middle using a streaming or batch application, and finally, write the output to a different system. That makes Structured Streaming an excellent choice for building ETL pipelines and enterprise integration use cases.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2VdCsIkg1zGXx7KZ5pKMxs","type":"Asset","createdAt":"2022-03-22T15:33:30.767Z","updatedAt":"2022-09-25T08:16:26.514Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"DataFrames formats and sources for Apache Spark","description":"Image showing logos of DataFrames formats and sources for Spark with Kafka","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2VdCsIkg1zGXx7KZ5pKMxs/46ff573d640a1d98833fad1c7abeb747/Screen_Shot_2022-03-22_at_10.33.19_AM.png","details":{"size":156841,"image":{"width":1320,"height":468}},"fileName":"Screen Shot 2022-03-22 at 10.33.19 AM.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Use cases for Kafka and Spark","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As we discussed, Kafka and Spark integrate very well via Structured Streaming. While Kafka provides scalable and durable storage for streaming data, Spark provides fault-tolerant computation on the data coming from Kafka. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Combining these technologies allows us to build large-scale data processing use cases.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Exploratory data analysis:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Data scientists and analysts prefer working with notebooks as they provide ad-hoc querying and interactive data visualization capabilities. Spark integrates well with popular notebooks like ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://zeppelin.apache.org/docs/latest/interpreter/spark.html"},"content":[{"nodeType":"text","value":"Zeppelin","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" and ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://jupyter.org/"},"content":[{"nodeType":"text","value":"Jupyter","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" as an interpreter to perform fast, on-demand computations on large data sets.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Stream processing applications: ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" With Structured Streaming, applications can process data from Kafka as it arrives. That makes Spark ideal for building real-time data processing use cases such as anomaly detection, real-time personalization, event-driven Microservices, and streaming ETL pipelines.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Batch analytics","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" - Spark can process data stored in Kafka as a batch. Typically, historical events stored in a compacted Kafka topic can populate new databases, backfill existing systems and run analytical workloads.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Training machine learning models:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Spark can be combined with ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://spark.apache.org/docs/latest/ml-guide.html#:~:text=MLlib%20is%20Spark's%20machine%20learning,regression%2C%20clustering%2C%20and%20collaborative%20filtering"},"content":[{"nodeType":"text","value":"MLlib","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" to train machine learning models with data coming from Kafka, either in real-time or batch mode.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data pipelines","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" - Spark’s ability to consume data from Kafka as a stream or batch allows for building ETL data pipelines.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This article introduces two Apache projects that have been popular in the big data processing space, Apache Kafka and Apache Spark. It gives you an overview of their architecture and the areas they overlap in functionality. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The key takeaway here is that Kafka and Spark are not two technologies that compete. Instead, they complement each other well with Structured Streaming. Having that in mind will always help you make better decisions when planning your next large-scale data-intensive application.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If you're looking for a way to streamline Apache Kafka management, ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"try Conduktor today","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2SX6zpOndgjWDVop0j59Uz","type":"Entry","createdAt":"2021-10-08T17:41:29.513Z","updatedAt":"2022-09-27T10:40:02.619Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":6,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Stéphane","surname":"Maarek","title":"Chief Evangelist","twitter":"https://twitter.com/StephaneMaarek","github":"https://github.com/simplesteph","linkedIn":"https://www.linkedin.com/in/stephanemaarek/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7y6tdCR7qU2yEWMJmQDS23","type":"Asset","createdAt":"2021-10-08T17:40:54.512Z","updatedAt":"2022-09-25T07:11:34.678Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Stephane Maarek","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7y6tdCR7qU2yEWMJmQDS23/523069a652132097fc9825d7831c0790/stephane-m.png","details":{"size":47223,"image":{"width":192,"height":192}},"fileName":"stephane-m.png","contentType":"image/png"}}}}},"publishDate":"2022-03-23T17:00+00:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5ozRWHw5b0druIRnN7joNo","type":"Entry","createdAt":"2022-02-13T19:50:51.569Z","updatedAt":"2022-09-25T13:51:31.789Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Capturing MySQL database changes using Debezium, Kafka, and Conduktor","description":"In this post, we’re going to look at using Kafka Connect in Conduktor and specifically how to use Debezium to monitor the changes in a MySQL database.","noIndex":false}},"slug":"capturing-mysql-database-changes-using-debezium-kafka-and-conduktor","title":"Capturing MySQL database changes using Debezium, Kafka, and Conduktor","description":"Using Kafka Connect in Conduktor and specifically how to use Debezium to monitor the changes in a MySQL database.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2nPKXiBRnxLWrgPPL61urC","type":"Asset","createdAt":"2022-02-13T19:50:28.460Z","updatedAt":"2022-09-25T16:54:30.355Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Capturing MySQL database changes using Debezium, Kafka, and Conduktor","description":"MySQL database changes using Debezium, Kafka, and Conduktor","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2nPKXiBRnxLWrgPPL61urC/fbd9b192804f8b7a8f2378b50f87d9e4/Capturing_MySQL_database_changes_using_Debezium__Kafka__and_Conduktor.jpg","details":{"size":449772,"image":{"width":1920,"height":1280}},"fileName":"Capturing MySQL database changes using Debezium, Kafka, and Conduktor.jpg","contentType":"image/jpeg"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is Conduktor?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor is a native desktop application that is a graphical user interface for your Apache Kafka environment and supports the full Kafka ecosystem including Schema Registry, Kafka Connect, Kafka Streams, and ksqlDB.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If you use Apache Kafka, Conduktor will increase productivity, reduce costs and ultimately, accelerate delivery of your critical data streaming projects. ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Download Conduktor now","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is Kafka Connect?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/features/kafka-connect/"},"content":[{"nodeType":"text","value":"Kafka Connect","marks":[],"data":{}}]},{"nodeType":"text","value":" is a tool in the Apache Kafka ecosystem which allows users to integrate their data sources between Apache Kafka and other data systems in a reliable way.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka Connect makes it easy to quickly start a connector and integrate real-time data either into Kafka from a source connector or out of Kafka into a sink connector of some kind. You can learn more about Kafka connect and its uses cases in this ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/why-use-kafka-connect/"},"content":[{"nodeType":"text","value":"blog post by Stephane Derosiaux","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1v7OBTO0mZ4GRjEcD4C1LI","type":"Asset","createdAt":"2022-02-08T12:34:40.120Z","updatedAt":"2023-01-30T13:48:30.853Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Kafka Connect Architecture","description":"Diagram showing basic Kafka Connect architecture","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1v7OBTO0mZ4GRjEcD4C1LI/eeefa4ae8acb15967a0e7ad058d754d7/Kafka_Connect_Architecture.png","details":{"size":37984,"image":{"width":2882,"height":648}},"fileName":"Kafka Connect Architecture.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is Debezium?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://debezium.io/"},"content":[{"nodeType":"text","value":"Debezium","marks":[],"data":{}}]},{"nodeType":"text","value":" is essentially a modern, distributed open source change data capture (CDC) platform that supports the monitoring of a number of database systems.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Debezium is built on top of Apache Kafka and provides Kafka Connect compatible connectors that monitor specific database management systems. It turns your existing databases into event streams, so applications can see and respond immediately to each row-level change in the databases.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Debezium records the history of data changes in Kafka logs, from where your application consumes them. This makes it possible for your application to easily consume all of the events correctly and completely. Even if your application stops unexpectedly, it will not miss anything: when the application restarts, it will resume consuming the events where it left off. As the data in the database changes, you will see the resulting change in an event stream.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7lNmJn2RQDJGQ2BqvgRJBp","type":"Asset","createdAt":"2022-02-16T12:12:56.881Z","updatedAt":"2023-01-30T13:54:56.863Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"debezium-architecture","description":"Debezium in action","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7lNmJn2RQDJGQ2BqvgRJBp/2ff07e7408b0b29c49e4d4ce2ce0be66/debezium-architecture.png","details":{"size":72807,"image":{"width":1712,"height":389}},"fileName":"debezium-architecture.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[{"type":"italic"}],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://debezium.io/documentation/reference/stable/architecture.html"},"content":[{"nodeType":"text","value":"Source","marks":[{"type":"italic"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Why Debezium?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Thanks to Kafka Connect and Debezium, change data capture, or CDC is now a common pattern that allows you to expose database changes as events into Kafka. As your database has a change occur you can track that directly in Kafka.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As illustrated in figure 2, one method of achieving this is by capturing the changelogs of a database upstream in either a Postgres or MySql database using the Debezium Kafka connectors. The changelog itself can be stored in Kafka, where a series of deployed programs are able to transform, aggregate, and join the data together. The processed data can then be streamed out to a sink database such as ElasticSearch or a data warehouse.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Debezium Use Cases","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"According to Gunnar Morling from Redhat, tech lead for Debezium, CDC means \"liberation for your data\". He goes into more detail on the plans for Debezium going forward in ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://speakerdeck.com/gunnarmorling/practical-change-data-streaming-use-cases-with-apache-kafka-and-debezium-qcon-san-francisco-2019"},"content":[{"nodeType":"text","value":"this talk","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ultimately, Debezium lets you track data changes, replicate data, update caches, sync data between microservices, and create audit logs among much more.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Tutorial","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As an example, we are going to look at utilizing Debezium in Conduktor and accomplishing this with Docker. We will carry out the following four steps.","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Step 1: Start Apache Kafka, Kafka Connect, and Debezium with Docker","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Step 2: Open Conduktor","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Step 3: Add MYSQL Debezium connector in Conduktor","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Step 4: View all created topics in Conduktor","marks":[],"data":{}}]}]}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Prerequisites","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Before we can carry out the above we need to have some prerequisites completed as shown below.","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Docker is installed and running. This tutorial uses Docker. For more information, see ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.docker.com/engine/installation/"},"content":[{"nodeType":"text","value":"the Docker Engine installation documentation","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":".","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Java 11+ installed. You can download it from ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.oracle.com/java/technologies/downloads/"},"content":[{"nodeType":"text","value":"here","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka installed. You can download Kafka from ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/downloads"},"content":[{"nodeType":"text","value":"here","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":".","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor is installed and running. You can follow ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/sign-in-section/install"},"content":[{"nodeType":"text","value":"these instructions","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":" to get started.","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For a helpful video on installing conduktor, Docker, and other prerequisites please see this from one of our founders Stephane Maarek. ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.youtube.com/watch?v=2sTCZeREs8w"},"content":[{"nodeType":"text","value":"Starting a local Apache Kafka Cluster on Conduktor in 3 minutes using Docker","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Step 1: Start Apache Kafka, Kafka Connect, and Debezium with Docker","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We want to use ","marks":[],"data":{}},{"nodeType":"text","value":"the docker-compose","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" file below to start:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A Kafka broker instance","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A Zookeeper instance","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A Kafka Connect instance","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A Mysql server","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A Debezium connector for Mysql","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"i) Download the YAML file","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To do this we will download the ","marks":[],"data":{}},{"nodeType":"text","value":"YAML file below","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" and save it as docker-compose.yml.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"YAML file","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3AR8i6FxYKVrJEOLwsdMTP","type":"Entry","createdAt":"2023-01-30T13:52:18.313Z","updatedAt":"2023-01-30T13:52:54.845Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"yaml","code":"---\nversion: '2'\n\nservices:\n zookeeper:\n image: confluentinc/cp-zookeeper:6.0.0\n hostname: zookeeper\n container_name: zookeeper\n ports:\n - \"2181:2181\"\n environment:\n ZOOKEEPER_CLIENT_PORT: 2181\n ZOOKEEPER_TICK_TIME: 2000\n\n broker:\n image: confluentinc/cp-kafka:6.0.0\n hostname: broker\n container_name: broker\n depends_on:\n - zookeeper\n ports:\n - \"29092:29092\"\n - \"9999:9999\"\n environment:\n KAFKA_BROKER_ID: 1\n KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'\n KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT\n KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092\n KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1\n KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1\n KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1\n KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0\n KAFKA_JMX_PORT: 9999\n KAFKA_JMX_HOSTNAME: localhost\n\n connect:\n image: confluentinc/cp-kafka-connect:latest\n hostname: connect\n container_name: connect\n depends_on:\n - broker\n ports:\n - \"8083:8083\"\n command:\n - bash\n - -c\n - |\n confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:latest\n confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:latest\n confluent-hub install --no-prompt debezium/debezium-connector-mysql:latest\n /etc/confluent/docker/run\n environment:\n CONNECT_BOOTSTRAP_SERVERS: \"broker:9092\"\n CONNECT_REST_ADVERTISED_HOST_NAME: connect\n CONNECT_REST_PORT: 8083\n CONNECT_GROUP_ID: compose-connect-group\n CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs\n CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1\n CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000\n CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets\n CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1\n CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status\n CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1\n CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter\n CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter\n CONNECT_PLUGIN_PATH: \"/usr/share/java,/usr/share/confluent-hub-components\"\n CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR\n KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=connect -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=5555 -Dcom.sun.management.jmxremote.port=5555\n\n mysql:\n image: debezium/example-mysql:1.7\n hostname: mysql\n container_name: mysql\n depends_on:\n - broker\n environment:\n - MYSQL_ROOT_PASSWORD=debezium\n - MYSQL_USER=mysqluser\n - MYSQL_PASSWORD=mysqlpw\n ports:\n - '3306:3306'"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"ii) Run docker compose","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"After saving the above YAML file as ","marks":[],"data":{}},{"nodeType":"text","value":"docker-compose.yml","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":", open your command line interface or terminal and run the following command:","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7x6czi1yz6ffql5lMSnWYo","type":"Entry","createdAt":"2023-01-30T13:55:38.160Z","updatedAt":"2023-01-30T13:55:38.160Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"terminalBlock"}},"locale":"en-US"},"fields":{"command":"_docker-compose up -d_"}}},"content":[]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3LufGnIrQ6YVrPZlNQXcXS","type":"Asset","createdAt":"2022-02-15T06:05:21.007Z","updatedAt":"2023-01-30T13:56:49.546Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Kafka Debezium Console Screenshot 1","description":"Kafka Debezium Console Screenshot","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3LufGnIrQ6YVrPZlNQXcXS/0c8876b7ec4f8a300f7288d0b87ac639/Kafka_Debezium_Console_Screenshot_1.png","details":{"size":82951,"image":{"width":1240,"height":742}},"fileName":"Kafka Debezium Console Screenshot 1.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"iii) Check its running","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We will just check that everything is up and running on docker we will enter the following command:","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4bDgS9IeWB5JnKfM40fJWw","type":"Entry","createdAt":"2023-01-30T13:55:51.419Z","updatedAt":"2023-01-30T13:55:51.419Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"terminalBlock"}},"locale":"en-US"},"fields":{"command":"_Docker-compose ps_"}}},"content":[]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1X1OZhvsP4suC2Bt3q8mEf","type":"Asset","createdAt":"2022-02-15T06:06:04.312Z","updatedAt":"2023-01-30T13:57:06.675Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Kafka Debezium Console Screenshot 2","description":"Kafka Debezium Console Screenshot 2","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1X1OZhvsP4suC2Bt3q8mEf/3c0fa56cad1700a95a5f3cac2ffe796e/Kafka_Debezium_Console_Screenshot_2.png","details":{"size":141391,"image":{"width":1240,"height":742}},"fileName":"Kafka Debezium Console Screenshot 2.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Step 2: Opening Conduktor","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Moving to the next step we will open up Conduktor and configure the cluster from our Docker compose file.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"i) Open Conduktor and sign in","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1cQkOsjSzYquyXfHJb7gPD","type":"Asset","createdAt":"2022-02-16T12:16:15.225Z","updatedAt":"2023-01-30T13:57:55.649Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Kafka Debezium Conduktor 3","description":"Kafka Debezium Conduktor","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1cQkOsjSzYquyXfHJb7gPD/0f5477c124bbe952a0eceb56aae72a7b/Kafka_Debezium_Conduktor_3.png","details":{"size":109360,"image":{"width":1212,"height":700}},"fileName":"Kafka Debezium Conduktor 3.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"ii) Configuring your Kafka Cluster","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Enter the bootstrap server for the Kafka cluster as localhost:29092 as specified in the YAML file. The cluster name can be anything.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4sVwH7AOCzB1LinUL9XdFs","type":"Asset","createdAt":"2022-02-16T12:17:03.857Z","updatedAt":"2023-01-30T13:58:32.376Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Kafka Debezium Conduktor 4","description":"Kafka Debezium Conduktor","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4sVwH7AOCzB1LinUL9XdFs/e6ef3bb3b0d9c10a1cfdaf0f3f9b98d2/Kafka_Debezium_Conduktor_4.png","details":{"size":355834,"image":{"width":1214,"height":794}},"fileName":"Kafka Debezium Conduktor 4.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"\nNext, enter the URL for the instance of Kafka connect as ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"http://localhost:8083/"},"content":[{"nodeType":"text","value":"http://localhost:8083","marks":[],"data":{}}]},{"nodeType":"text","value":" as specified in the YAML file.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2haaCD6OfQFdV0LM7s9lbI","type":"Asset","createdAt":"2022-02-16T12:18:33.594Z","updatedAt":"2023-01-30T13:58:52.583Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Kafka Debezium Conduktor 5","description":"Kafka Debezium Conduktor","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2haaCD6OfQFdV0LM7s9lbI/6eb4c71de9afe50c701dc02ce7bea6e8/Kafka_Debezium_Conduktor_5.png","details":{"size":316492,"image":{"width":1215,"height":794}},"fileName":"Kafka Debezium Conduktor 5.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"\nClick on the ","marks":[],"data":{}},{"nodeType":"text","value":"Save","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" button to save the cluster configuration.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Then click on the cluster itself to connect.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2ER1B1dT9uolGoSULwRhmG","type":"Asset","createdAt":"2022-02-16T12:19:15.809Z","updatedAt":"2023-01-30T13:59:20.642Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Kafka Debezium Conduktor 6","description":"Kafka Debezium Conduktor","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2ER1B1dT9uolGoSULwRhmG/ea1f4e23ccb5799f38cbff58ebb202b1/Kafka_Debezium_Conduktor_6.png","details":{"size":272371,"image":{"width":1215,"height":699}},"fileName":"Kafka Debezium Conduktor 6.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You will come to the ","marks":[],"data":{}},{"nodeType":"text","value":"Overview","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" tab inside Conduktor with a view similar to below","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1qDIsvORfLThJng7BqyPXT","type":"Asset","createdAt":"2022-02-16T12:19:54.164Z","updatedAt":"2023-01-30T13:59:50.826Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Kafka Debezium Conduktor 7","description":"Kafka Debezium Conduktor","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1qDIsvORfLThJng7BqyPXT/7f3ea3191d544d973995a11db9db0037/Kafka_Debezium_Conduktor_7.png","details":{"size":371908,"image":{"width":1067,"height":781}},"fileName":"Kafka Debezium Conduktor 7.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Step 3: Adding a Debezium Connector for Kafka Connect in Conduktor","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"i) Kafka Connect Tab","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"After configuring our cluster we want to add our Debezium connector in Conduktor.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To do this we will go to the Kafka Connect tab as shown below.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4xKBryc8uLxLu3LMRZf0kF","type":"Asset","createdAt":"2022-02-16T12:21:01.531Z","updatedAt":"2023-01-30T14:00:40.294Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Kafka Debezium Conduktor 8","description":"Kafka Debezium Conduktor","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4xKBryc8uLxLu3LMRZf0kF/98b2126dffb2acd4d277fe0469ee039e/Kafka_Debezium_Conduktor_8.png","details":{"size":283075,"image":{"width":1067,"height":781}},"fileName":"Kafka Debezium Conduktor 8.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"\n","marks":[],"data":{}},{"nodeType":"text","value":"ii) Creating a connector in Conduktor","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We now have two options to add a connector we can","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Option 1:Using the connect configuration wizard in Conduktor","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Option 2: Import the connector in JSON format directly","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Option 1:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Using the connect configuration wizard we will click on ","marks":[],"data":{}},{"nodeType":"text","value":"io.debezium.connector.mysql.MySqlConnector:1.7.1.Final","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" and enter the information required in each of the connector wizard tabs.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3CIiuOrCGy7w0kSSQXV5FZ","type":"Asset","createdAt":"2022-02-16T12:21:34.094Z","updatedAt":"2023-01-30T14:01:12.554Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Kafka Debezium Conduktor 9","description":"Kafka Debezium Conduktor","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3CIiuOrCGy7w0kSSQXV5FZ/837b52812dbc0d35e38876f434b7c04f/Kafka_Debezium_Conduktor_9.png","details":{"size":363056,"image":{"width":1068,"height":781}},"fileName":"Kafka Debezium Conduktor 9.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"\nLeave the default values for wizard tabs, including Transforms, Predicates, Error Handling, and Topic Creation. When the MySQL tab shows up, provide the following information.","marks":[],"data":{}}]},{"nodeType":"table","data":{},"content":[{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-header-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Field","marks":[],"data":{}}]}]},{"nodeType":"table-header-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Value","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Hostname","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"mysql","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Port","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"3306","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"User","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"debezium","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Password","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"dbz","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Namespace","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"dbserver1","marks":[],"data":{}}]}]}]}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2WkXPFbtwhFAC0qahCML75","type":"Asset","createdAt":"2022-02-16T12:22:44.082Z","updatedAt":"2023-01-30T14:01:57.691Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Kafka Debezium Conduktor 11","description":"Kafka Debezium Conduktor","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2WkXPFbtwhFAC0qahCML75/0166712d65f920dc90fdc57132db6e84/Kafka_Debezium_Conduktor_11.png","details":{"size":368674,"image":{"width":1070,"height":782}},"fileName":"Kafka Debezium Conduktor 11.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In the ","marks":[],"data":{}},{"nodeType":"text","value":"History Storage","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" tab, enter the following information.","marks":[],"data":{}}]},{"nodeType":"table","data":{},"content":[{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-header-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Field","marks":[],"data":{}}]}]},{"nodeType":"table-header-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Value","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka broker addresses","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"broker:9092","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Database history topic name","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"schema-changes.inventory","marks":[],"data":{}}]}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"\nLeave Events and Custom Config tabs with the defaults. Finally, review the connector configurations before applying them to the cluster.\n","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Option 2:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Using the connector configuration in JSON format, we will copy and paste it directly into Conduktor.\n\nFrom this screen onwards, continue the wizard with the default values in the tabs.","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1tZrwE0DmY8xcccxHr68b8","type":"Entry","createdAt":"2023-01-30T14:03:31.688Z","updatedAt":"2023-01-30T14:03:31.688Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"json","code":"{\n \"name\": \"inventory-connector\",\n \"config\": {\n \"connector.class\": \"io.debezium.connector.mysql.MySqlConnector\",\n \"tasks.max\": \"1\",\n \"database.hostname\": \"mysql\",\n \"database.port\": \"3306\",\n \"database.user\": \"debezium\",\n \"database.password\": \"dbz\",\n \"database.server.id\": \"6400\",\n \"database.server.name\": \"dbserver1\",\n \"database.include.list\": \"inventory\",\n \"database.history.kafka.bootstrap.servers\": \"broker:9092\",\n \"database.history.kafka.topic\": \"schema-changes.inventory\"\n }\n}"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"\n","marks":[],"data":{}},{"nodeType":"text","value":"iii) View Connector running","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We can now see our Debezium MySql connector with its current status and all topics. Initially, it might take a few seconds for the topics to get created.\n","marks":[],"data":{}},{"nodeType":"text","value":"Step 4: View all created topics in Conduktor","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Open the Topics tab in Conduktor to see all of the newly created topics that will show the change data capture as part of the debezium connector.\n\nAfter we have added the Debezium MySQL connector in Conduktor we can see it starts monitoring the database for data change events, in this case, it is a database for inventory with specific topics related to different tables.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The events are written to the following topics with the ","marks":[],"data":{}},{"nodeType":"text","value":"dbserver1","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" prefix (the name of the connector):","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"dbserver1:","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":" The schema change topic to which all of the Data definition language (DDL) statements are written.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"dbserver1.inventory.products:","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":" Captures change events for the products table in the inventory database.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"dbserver1.inventory.products_on_hand:","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":" Captures change events for the products_on_hand table in the inventory database.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"dbserver1.inventory.customers:","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":" Captures change events for the customers table in the inventory database.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"dbserver1.inventory.orders:","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":" Captures change events for the orders table in the inventory database.","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We can also look further into one of the topics by opening a consumer directly in Conduktor. For this example, we will explore the dbserver1.inventory.addresses topic.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To do this click on “Consumer” on the top right corner to create a new consumer.\n\nIn the top left corner of this tab click on “Pick a Topic to inspect its data” and then choose db.server1.inventory.addresses.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We will then choose “Start from The Beginning” and click on “Start” on the bottom right. This will show the different types of change events associated with addresses as follows.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We have now run an Apache Kafka broker, a Zookeeper broker, Kafka Connect, MySql, and Debezium through docker and created a debezium connector directly through Conduktor and seen the changes illustrated in a consumer.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As shown in this tutorial Conduktor makes it easier for you to manage your instances of Kafka Connect, add and edit them and when using Debezium you can keep track of all topics involved very easily and quickly and filter by time/date or offset.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"References","marks":[],"data":{}}]},{"nodeType":"ordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://debezium.io/documentation/reference/stable/architecture.html"},"content":[{"nodeType":"text","value":"https://debezium.io/documentation/reference/stable/architecture.html","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://subscription.packtpub.com/book/big_data_and_business_intelligence/9781787281349/7/ch07lvl1sec80/kafka-connect"},"content":[{"nodeType":"text","value":"https://subscription.packtpub.com/book/big_data_and_business_intelligence/9781787281349/7/ch07lvl1sec80/kafka-connect","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/features/kafka-connect/features/"},"content":[{"nodeType":"text","value":"https://docs.conduktor.io/features/kafka-connect/features","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/why-use-kafka-connect/"},"content":[{"nodeType":"text","value":"https://www.conduktor.io/why-use-kafka-connect","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.sderosiaux.com/articles/2020/01/06/learnings-from-using-kafka-connect-debezium-postgresql/"},"content":[{"nodeType":"text","value":"https://www.sderosiaux.com/articles/2020/01/06/learnings-from-using-kafka-connect-debezium-postgresql/","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://debezium.io/documentation/reference/stable/tutorial.html"},"content":[{"nodeType":"text","value":"https://debezium.io/documentation/reference/stable/tutorial.html","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://debezium.io/documentation/reference/stable/tutorial.html"},"content":[{"nodeType":"text","value":"https://debezium.io/documentation/reference/stable/tutorial.html","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/documentation/#connectconfigs"},"content":[{"nodeType":"text","value":"https://kafka.apache.org/documentation/#connectconfigs","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://speakerdeck.com/gunnarmorling/practical-change-data-streaming-use-cases-with-apache-kafka-and-debezium-qcon-san-francisco-2019"},"content":[{"nodeType":"text","value":"https://speakerdeck.com/gunnarmorling/practical-change-data-streaming-use-cases-with-apache-kafka-and-debezium-qcon-san-francisco-2019","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}}]}]},"body":"## What is Conduktor?\n\nConduktor is a native desktop application that is a graphical user interface for your Apache Kafka environment and supports the full Kafka ecosystem including Schema Registry, Kafka Connect, Kafka Streams, and ksqlDB.\n\nIf you use Apache Kafka, Conduktor will increase productivity, reduce costs and ultimately, accelerate delivery of your critical data streaming projects. [Download Conduktor now](https://www.conduktor.io/download)\n\n## What is Kafka Connect?\n\n[Kafka Connect](https://docs.conduktor.io/features/kafka-connect) is a tool in the Apache Kafka ecosystem which allows users to integrate their data sources between Apache Kafka and other data systems in a reliable way. \n\nKafka Connect makes it easy to quickly start a connector and integrate real-time data either into Kafka from a source connector or out of Kafka into a sink connector of some kind. You can learn more about Kafka connect and its uses cases in this [blog post by Stephane Derosiaux](https://www.conduktor.io/why-use-kafka-connect/) \n\n![Kafka Connect Architecture](//images.ctfassets.net/lm7q36eaz111/5tueYlavyxgOJQogM61N5g/26bd63cdd7af1cdb4d774a368a1712a9/Kafka_Connect_Architecture.png)\n*Figure 1 - Kafka Connect Architecture*\n
\n\n## What is Debezium?\n\n[Debezium](https://debezium.io/) is essentially a modern, distributed open source change data capture (CDC) platform that supports the monitoring of a number of database systems. \n\nDebezium is built on top of Apache Kafka and provides Kafka Connect compatible connectors that monitor specific database management systems. It turns your existing databases into event streams, so applications can see and respond immediately to each row-level change in the databases.\n\nDebezium records the history of data changes in Kafka logs, from where your application consumes them. This makes it possible for your application to easily consume all of the events correctly and completely. Even if your application stops unexpectedly, it will not miss anything: when the application restarts, it will resume consuming the events where it left off. As the data in the database changes, you will see the resulting change in an event stream.\n\n![debezium-architecture](//images.ctfassets.net/lm7q36eaz111/7lNmJn2RQDJGQ2BqvgRJBp/2ff07e7408b0b29c49e4d4ce2ce0be66/debezium-architecture.png)\n\n*Figure 2: Debezium in action. [Source](https://debezium.io/documentation/reference/stable/architecture.html)*\n
\n\n## Why Debezium?\n\nThanks to Kafka Connect and Debezium, change data capture, or CDC is now a common pattern that allows you to expose database changes as events into Kafka. As your database has a change occur you can track that directly in Kafka.\n\nAs illustrated in figure 2, one method of achieving this is by capturing the changelogs of a database upstream in either a Postgres or MySql database using the Debezium Kafka connectors. The changelog itself can be stored in Kafka, where a series of deployed programs are able to transform, aggregate, and join the data together. The processed data can then be streamed out to a sink database such as ElasticSearch or a data warehouse.\n\n## Debezium Use Cases\n\nAccording to Gunnar Morling from Redhat, tech lead for Debezium, CDC means \"liberation for your data\". He goes into more detail on the plans for Debezium going forward in [this talk](https://speakerdeck.com/gunnarmorling/practical-change-data-streaming-use-cases-with-apache-kafka-and-debezium-qcon-san-francisco-2019).\n\nUltimately, Debezium lets you track data changes, replicate data, update caches, sync data between microservices, and create audit logs among much more. \n\n## Tutorial\n\nAs an example, we are going to look at utilizing Debezium in Conduktor and accomplishing this with Docker. We will carry out the following four steps.\n\n- Step 1: Start Apache Kafka, Kafka Connect, and Debezium with Docker\n- Step 2: Open Conduktor\n- Step 3: Add MYSQL Debezium connector in Conduktor\n- Step 4: View all created topics in Conduktor\n\n### Prerequisites\n\nBefore we can carry out the above we need to have some prerequisites completed as shown below.\n\n- Docker is installed and running. This tutorial uses Docker. For more information, see [the Docker Engine installation documentation](https://docs.docker.com/engine/installation/).\n- Java 11+ installed. You can download it from [here](https://www.oracle.com/java/technologies/downloads/) \n- Apache Kafka installed. You can download Kafka from [here](https://kafka.apache.org/downloads). \n- Conduktor is installed and running. You can follow [these instructions](https://docs.conduktor.io/sign-in-section/install) to get started.\n\nFor a helpful video on installing conduktor, Docker, and other prerequisites please see this from one of our founders Stephane Maarek. [Starting a local Apache Kafka Cluster on Conduktor in 3 minutes using Docker](https://www.youtube.com/watch?v=2sTCZeREs8w)\n\n### Step 1: Start Apache Kafka, Kafka Connect, and Debezium with Docker\n\nWe want to use __the docker-compose__ file below to start:\n\n- A Kafka broker instance\n- A Zookeeper instance\n- A Kafka Connect instance\n- A Mysql server\n- A Debezium connector for Mysql\n\n__i) Download the YAML file__\n\nTo do this we will download the __YAML file below__ and save it as docker-compose.yml.\n\n### YAML file\n\n```yaml\n\n---\nversion: '2'\n\nservices:\n zookeeper:\n image: confluentinc/cp-zookeeper:6.0.0\n hostname: zookeeper\n container_name: zookeeper\n ports:\n - \"2181:2181\"\n environment:\n ZOOKEEPER_CLIENT_PORT: 2181\n ZOOKEEPER_TICK_TIME: 2000\n\n broker:\n image: confluentinc/cp-kafka:6.0.0\n hostname: broker\n container_name: broker\n depends_on:\n - zookeeper\n ports:\n - \"29092:29092\"\n - \"9999:9999\"\n environment:\n KAFKA_BROKER_ID: 1\n KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'\n KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT\n KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092\n KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1\n KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1\n KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1\n KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0\n KAFKA_JMX_PORT: 9999\n KAFKA_JMX_HOSTNAME: localhost\n\n connect:\n image: confluentinc/cp-kafka-connect:latest\n hostname: connect\n container_name: connect\n depends_on:\n - broker\n ports:\n - \"8083:8083\"\n command:\n - bash\n - -c\n - |\n confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:latest\n confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:latest\n confluent-hub install --no-prompt debezium/debezium-connector-mysql:latest\n /etc/confluent/docker/run\n environment:\n CONNECT_BOOTSTRAP_SERVERS: \"broker:9092\"\n CONNECT_REST_ADVERTISED_HOST_NAME: connect\n CONNECT_REST_PORT: 8083\n CONNECT_GROUP_ID: compose-connect-group\n CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs\n CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1\n CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000\n CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets\n CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1\n CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status\n CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1\n CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter\n CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter\n CONNECT_PLUGIN_PATH: \"/usr/share/java,/usr/share/confluent-hub-components\"\n CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR\n KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=connect -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=5555 -Dcom.sun.management.jmxremote.port=5555\n\n mysql:\n image: debezium/example-mysql:1.7\n hostname: mysql\n container_name: mysql\n depends_on:\n - broker\n environment:\n - MYSQL_ROOT_PASSWORD=debezium\n - MYSQL_USER=mysqluser\n - MYSQL_PASSWORD=mysqlpw\n ports:\n - '3306:3306'\n```\n
\n
\n\n__ii) Run docker compose__\n\nAfter saving the above YAML file as __docker-compose.yml__, open your command line interface or terminal and run the following command:\n\n _docker-compose up -d_\n\n![Kafka Debezium Console Screenshot 1](//images.ctfassets.net/lm7q36eaz111/3LufGnIrQ6YVrPZlNQXcXS/0c8876b7ec4f8a300f7288d0b87ac639/Kafka_Debezium_Console_Screenshot_1.png)\n\n__iii) Check its running__\n\nWe will just check that everything is up and running on docker we will enter the following command:\n\n _Docker-compose ps_\n\n![Kafka Debezium Console Screenshot 2](//images.ctfassets.net/lm7q36eaz111/1X1OZhvsP4suC2Bt3q8mEf/3c0fa56cad1700a95a5f3cac2ffe796e/Kafka_Debezium_Console_Screenshot_2.png)\n\n### Step 2: Opening Conduktor\n\nMoving to the next step we will open up Conduktor and configure the cluster from our Docker compose file.\n\n__i) Open Conduktor and sign in__\n![Kafka Debezium Conduktor 3](//images.ctfassets.net/lm7q36eaz111/6kPJmKnT1AETMqFzdKE9mk/d793cd01c67de57c85320d17f54b8a2c/Kafka_Debezium_Conduktor_3.png)\n__ii) Configuring your Kafka Cluster__\n\nEnter the bootstrap server for the Kafka cluster as localhost:29092 as specified in the YAML file. The cluster name can be anything.\n![Kafka Debezium Conduktor 4](//images.ctfassets.net/lm7q36eaz111/4sVwH7AOCzB1LinUL9XdFs/e6ef3bb3b0d9c10a1cfdaf0f3f9b98d2/Kafka_Debezium_Conduktor_4.png)\nNext, enter the URL for the instance of Kafka connect as http://localhost:8083 as specified in the YAML file.\n![Kafka Debezium Conduktor 5](//images.ctfassets.net/lm7q36eaz111/2haaCD6OfQFdV0LM7s9lbI/6eb4c71de9afe50c701dc02ce7bea6e8/Kafka_Debezium_Conduktor_5.png)\nClick on the **Save** button to save the cluster configuration.\n\nThen click on the cluster itself to connect.\n![Kafka Debezium Conduktor 6](//images.ctfassets.net/lm7q36eaz111/2ER1B1dT9uolGoSULwRhmG/ea1f4e23ccb5799f38cbff58ebb202b1/Kafka_Debezium_Conduktor_6.png)\nYou will come to the **Overview** tab inside Conduktor with a view similar to below\n![Kafka Debezium Conduktor 7](//images.ctfassets.net/lm7q36eaz111/1qDIsvORfLThJng7BqyPXT/7f3ea3191d544d973995a11db9db0037/Kafka_Debezium_Conduktor_7.png)\n### Step 3: Adding a Debezium Connector for Kafka Connect in Conduktor\n\n__i) Kafka Connect Tab__\n\nAfter configuring our cluster we want to add our Debezium connector in Conduktor.\n\nTo do this we will go to the Kafka Connect tab as shown below. \n![Kafka Debezium Conduktor 8](//images.ctfassets.net/lm7q36eaz111/4xKBryc8uLxLu3LMRZf0kF/98b2126dffb2acd4d277fe0469ee039e/Kafka_Debezium_Conduktor_8.png)\n__ii) Creating a connector in Conduktor__\n\nWe now have two options to add a connector we can\n\n- Option 1:Using the connect configuration wizard in Conduktor\n- Option 2: Import the connector in JSON format directly\n\n__Option 1:__ Using the connect configuration wizard we will click on __io.debezium.connector.mysql.MySqlConnector:1.7.1.Final__ and enter the information required in each of the connector wizard tabs.\n![Kafka Debezium Conduktor 9](//images.ctfassets.net/lm7q36eaz111/3CIiuOrCGy7w0kSSQXV5FZ/837b52812dbc0d35e38876f434b7c04f/Kafka_Debezium_Conduktor_9.png)\n![Kafka Debezium Conduktor 10](//images.ctfassets.net/lm7q36eaz111/1mzHRZxlutU6zW87Dj94ow/b1e40c0b0a4f3497204dfe5c35d7b787/Kafka_Debezium_Conduktor_10.png)\nLeave the default values for wizard tabs, including Transforms, Predicates, Error Handling, and Topic Creation. When the MySQL tab shows up, provide the following information.\n\n| Field | Value |\n|-----------|-----------|\n| Hostname | mysql |\n| Port | 3306 |\n| User | debezium |\n| Password | dbz |\n| Namespace | dbserver1 |\n\n![Kafka Debezium Conduktor 11](//images.ctfassets.net/lm7q36eaz111/2WkXPFbtwhFAC0qahCML75/0166712d65f920dc90fdc57132db6e84/Kafka_Debezium_Conduktor_11.png)\n\nIn the __History Storage__ tab, enter the following information.\n\n| Field | Value |\n|-----------------------------|--------------------------|\n| Kafka broker addresses | broker:9092 |\n| Database history topic name | schema-changes.inventory |\n\n![Kafka Debezium Conduktor 12](//images.ctfassets.net/lm7q36eaz111/5JiYYudmtn6nro1xCAsEkc/60476a80f0c2a10b068b95276871fb7e/Kafka_Debezium_Conduktor_12.png)\nLeave Events and Custom Config tabs with the defaults. Finally, review the connector configurations before applying them to the cluster.\n![Kafka Debezium Conduktor 13](//images.ctfassets.net/lm7q36eaz111/2VBgUXhcALqoHkoMJAaI0v/fdbdc3a4d2dd9c24db5f1f5b5e3d3516/Kafka_Debezium_Conduktor_13.png)\n\n__Option 2:__ Using the connector configuration in JSON format, we will copy and paste it directly into Conduktor.\n![Kafka Debezium Conduktor 14](//images.ctfassets.net/lm7q36eaz111/7wXRX71Ao0K3o8mZdXJD1h/310ea7dc23f686013c4dd0beef4107f8/Kafka_Debezium_Conduktor_14.png)\n![Kafka Debezium Conduktor 15](//images.ctfassets.net/lm7q36eaz111/1I6mAaWl4erS1Akg4t5YKf/268499c2802afd8babb2efdfa923a89c/Kafka_Debezium_Conduktor_15.png)\nFrom this screen onwards, continue the wizard with the default values in the tabs. \n\n```json\n{\n \"name\": \"inventory-connector\",\n \"config\": {\n \"connector.class\": \"io.debezium.connector.mysql.MySqlConnector\",\n \"tasks.max\": \"1\",\n \"database.hostname\": \"mysql\",\n \"database.port\": \"3306\",\n \"database.user\": \"debezium\",\n \"database.password\": \"dbz\",\n \"database.server.id\": \"6400\",\n \"database.server.name\": \"dbserver1\",\n \"database.include.list\": \"inventory\",\n \"database.history.kafka.bootstrap.servers\": \"broker:9092\",\n \"database.history.kafka.topic\": \"schema-changes.inventory\"\n }\n}\n\n```\n
\n
\n\n__iii) View Connector running__\n\nWe can now see our Debezium MySql connector with its current status and all topics. Initially, it might take a few seconds for the topics to get created.\n![Kafka Debezium Conduktor 16](//images.ctfassets.net/lm7q36eaz111/3oCjZsVSnB6ppP1RXpdgng/c86f6c533b906bc3a3b171c37d9b5eff/Kafka_Debezium_Conduktor_16.png)\n__Step 4: View all created topics in Conduktor__\n\nOpen the Topics tab in Conduktor to see all of the newly created topics that will show the change data capture as part of the debezium connector.\n![Kafka Debezium Conduktor 17](//images.ctfassets.net/lm7q36eaz111/1MLcoHHIFqcj5VNDbjdmBo/24dd787a69e410b0878380efe951f274/Kafka_Debezium_Conduktor_17.png)\nAfter we have added the Debezium MySQL connector in Conduktor we can see it starts monitoring the database for data change events, in this case, it is a database for inventory with specific topics related to different tables.\n\nThe events are written to the following topics with the __dbserver1__ prefix (the name of the connector):\n\n- __dbserver1:__ The schema change topic to which all of the Data definition language (DDL) statements are written.\n- __dbserver1.inventory.products:__ Captures change events for the products table in the inventory database.\n- __dbserver1.inventory.products_on_hand:__ Captures change events for the products_on_hand table in the inventory database.\n- __dbserver1.inventory.customers:__ Captures change events for the customers table in the inventory database.\n- __dbserver1.inventory.orders:__ Captures change events for the orders table in the inventory database.\n\nWe can also look further into one of the topics by opening a consumer directly in Conduktor. For this example, we will explore the dbserver1.inventory.addresses topic.\n\nTo do this click on “Consumer” on the top right corner to create a new consumer.\n![Kafka Debezium Conduktor 18](//images.ctfassets.net/lm7q36eaz111/SVNRKMIKcKelKMHrQZdot/62ee879c1af002520f06be5ef6dace16/Kafka_Debezium_Conduktor_18.png)\nIn the top left corner of this tab click on “Pick a Topic to inspect its data” and then choose db.server1.inventory.addresses.\n\n![Kafka Debezium Conduktor 19](//images.ctfassets.net/lm7q36eaz111/5JCkPxdmiijXUY6bkh6J4A/85a761bc0fadc63af742f5c181a0a8e6/Kafka_Debezium_Conduktor_19.png)\n\nWe will then choose “Start from The Beginning” and click on “Start” on the bottom right. This will show the different types of change events associated with addresses as follows.\n\n![Kafka Debezium Conduktor 20](//images.ctfassets.net/lm7q36eaz111/7EjCyzNMXLGnSoCJCKGI7l/9c3776c0c759f914159e4ae0e431b290/Kafka_Debezium_Conduktor_20.png)\n\n## Conclusion\n\nWe have now run an Apache Kafka broker, a Zookeeper broker, Kafka Connect, MySql, and Debezium through docker and created a debezium connector directly through Conduktor and seen the changes illustrated in a consumer.\n\nAs shown in this tutorial Conduktor makes it easier for you to manage your instances of Kafka Connect, add and edit them and when using Debezium you can keep track of all topics involved very easily and quickly and filter by time/date or offset. \n\n## References\n\n1. [https://debezium.io/documentation/reference/stable/architecture.html](https://debezium.io/documentation/reference/stable/architecture.html)\n2. [https://subscription.packtpub.com/book/big_data_and_business_intelligence/9781787281349/7/ch07lvl1sec80/kafka-connect](https://subscription.packtpub.com/book/big_data_and_business_intelligence/9781787281349/7/ch07lvl1sec80/kafka-connect)\n3. [https://docs.conduktor.io/features/kafka-connect/features](https://docs.conduktor.io/features/kafka-connect/features)\n4. [https://www.conduktor.io/why-use-kafka-connect](https://www.conduktor.io/why-use-kafka-connect)\n5. [https://www.sderosiaux.com/articles/2020/01/06/learnings-from-using-kafka-connect-debezium-postgresql/](https://www.sderosiaux.com/articles/2020/01/06/learnings-from-using-kafka-connect-debezium-postgresql/)\n6. [https://debezium.io/documentation/reference/stable/tutorial.html](https://debezium.io/documentation/reference/stable/tutorial.html)\n7. [https://debezium.io/documentation/reference/stable/tutorial.html](https://debezium.io/documentation/reference/stable/tutorial.html)\n8. [https://kafka.apache.org/documentation/#connectconfigs](https://kafka.apache.org/documentation/#connectconfigs)\n9. [https://speakerdeck.com/gunnarmorling/practical-change-data-streaming-use-cases-with-apache-kafka-and-debezium-qcon-san-francisco-2019](https://speakerdeck.com/gunnarmorling/practical-change-data-streaming-use-cases-with-apache-kafka-and-debezium-qcon-san-francisco-2019)","author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3GpYm8e8ScX12WYASfZqde","type":"Entry","createdAt":"2021-11-05T09:29:35.518Z","updatedAt":"2021-11-05T09:29:35.518Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Lorcan","title":"Customer Engineer","linkedIn":"https://www.linkedin.com/in/lorcan-cooke-b5aa77a2/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"14A3qRf2QTdXJx4nDL3rCD","type":"Asset","createdAt":"2021-11-05T09:29:26.913Z","updatedAt":"2022-09-25T07:29:50.969Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Lorcan","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/14A3qRf2QTdXJx4nDL3rCD/79fbd3877631e7e2942459599edb7d6a/lorcan_2x.png","details":{"size":44928,"image":{"width":192,"height":192}},"fileName":"lorcan@2x.png","contentType":"image/png"}}}}},"publishDate":"2022-02-14T17:00+00:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6difQWg1XrjEX9phVuYzZJ","type":"Entry","createdAt":"2022-02-08T11:51:38.349Z","updatedAt":"2022-09-25T13:51:05.997Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Apache Kafka vs Amazon Kinesis: : Comparing Data Streaming Systems Across 5 Dimensions","description":"Apache Kafka vs. AWS Kinesis - Comparing data event streaming options on architecture, admin, data retention, data replication, integration and pricing ","noIndex":false}},"slug":"apache-kafka-vs-amazon-kinesis-comparing-across-five-dimensions","title":"Kafka vs Kinesis: Comparing Across Five Dimensions","description":"In this blog, we deep-dive into Kinesis vs Kafka, evaluating the data event streaming solutions across 5 dimensions.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4GI3zI20NMgFxPkgzlv7vt","type":"Asset","createdAt":"2022-02-08T12:02:33.593Z","updatedAt":"2022-09-25T16:55:04.778Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Kinesis vs Kafka - Conduktor Blog","description":"Kinesis vs Kafka image showing systems architect outside server room","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4GI3zI20NMgFxPkgzlv7vt/dea4e6b6fe50af4482675918b6cdc22a/Kinesis_vs_Kafka_-_Conduktor_Blog.jpg","details":{"size":449316,"image":{"width":1920,"height":1282}},"fileName":"Kinesis vs Kafka - Conduktor Blog.jpg","contentType":"image/jpeg"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Gone are the days when organizations used to make decisions based on emotions and experience. With the inception of modern data analytics and machine learning technologies, organizations are transitioning into data-driven, real-time business decision-making.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Technologies for collecting, storing, and processing business events in real-time are getting more critical than ever. Apache Kafka and AWS Kinesis are two event streaming platforms that enable ingesting a large number of events each second and storing them durably until they are analyzed.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This article compares Kafka and Kinesis across five dimensions. Hopefully, it will provide you with a useful reference for picking between them in the future.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Apache Kafka - An overview","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/"},"content":[{"nodeType":"text","value":"Kafka","marks":[],"data":{}}]},{"nodeType":"text","value":" was originally developed at LinkedIn as a publish-subscribe system. But, it has evolved into a distributed, fault-tolerant, and scalable event streaming platform.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka is designed to operate as a distributed system that could span multiple data centers. A Kafka cluster is a collection of brokers who organize events into topics and store them durably for a configurable amount of time.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka provides a set of APIs for reading and writing data. Applications such as web applications, IoT devices, and Microservices could use the ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/documentation/#producerapi"},"content":[{"nodeType":"text","value":"Producer API","marks":[],"data":{}}]},{"nodeType":"text","value":" to write events into a Kafka topic. Consumer applications like stream processors and analytics databases subscribe to a topic and read events using ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/documentation/#consumerapi"},"content":[{"nodeType":"text","value":"Consumer API","marks":[],"data":{}}]},{"nodeType":"text","value":". Multiple producers can simultaneously produce events to a topic while many consumers already consume from the same topic. That leads to virtually unlimited scalability.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"mSYc3d9f4n5lKQ8W4GC0g","type":"Asset","createdAt":"2022-02-08T12:10:55.338Z","updatedAt":"2022-09-25T08:15:59.579Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Apache Kafka Architecture","description":"Diagram showing the basic architecture of Apache Kafka vs Amazon Kinesis.","file":{"url":"//images.ctfassets.net/lm7q36eaz111/mSYc3d9f4n5lKQ8W4GC0g/d01b03f76f0f113896ece90b490bba83/Apache_Kafka_Architecture.png","details":{"size":89744,"image":{"width":2880,"height":2012}},"fileName":"Apache Kafka Architecture.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka was donated to the Apache Software Foundation (ASF) to become an open-source project. That means you can download, install, and manage an Apache Kafka cluster free of charge. Also, Kafka is available as a managed service from vendors such as ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.confluent.io/"},"content":[{"nodeType":"text","value":"Confluent","marks":[],"data":{}}]},{"nodeType":"text","value":", ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://aws.amazon.com/msk/"},"content":[{"nodeType":"text","value":"AWS MSK","marks":[],"data":{}}]},{"nodeType":"text","value":", and ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://aiven.io/"},"content":[{"nodeType":"text","value":"Aiven","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"AWS Kinesis - An overview","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kinesis is an event streaming platform based on the publish-subscribe principle. Kinesis is offered as a managed service by AWS.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kinesis is the umbrella term used for four different services--Kinesis Data Streams, Kinesis Data Firehose, Kinesis Video Streams, and Kinesis Data Analytics. While each service serves a specific purpose, we will only consider Kinesis Data Streams for the comparison as it provides a foundation for the rest of the services. We will refer to Kinesis Data Streams as Kinesis for the sake of simplicity.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kinesis organizes its data records into ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.aws.amazon.com/streams/latest/dev/key-concepts.html#shard"},"content":[{"nodeType":"text","value":"shards","marks":[],"data":{}}]},{"nodeType":"text","value":". Like Kafka, Kinesis also provides its users with APIs for producing and consuming events into a shard. Those APIs are available in multiple programming languages and tightly integrated with AWS SDKs and the CLI.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4DzhpjqRuexCYlFAXXEod3","type":"Asset","createdAt":"2022-02-08T12:14:28.725Z","updatedAt":"2023-01-30T13:46:59.076Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Amazon Kinesis Architecture","description":"Diagram showing basic architecture of Amazon Kinesis vs Kafka","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4DzhpjqRuexCYlFAXXEod3/e73607eae7a1e712cb7e4c028b247d46/Amazon_Kinesis_Architecture.png","details":{"size":86536,"image":{"width":2880,"height":2012}},"fileName":"Amazon Kinesis Architecture.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Kafka vs. Kinesis - Dimensions for the comparison","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Both Kafka and Kinesis are prominent technologies in the event streaming space. A decision to choose either of them should be made rationally. Hence, we came up with the following dimensions to compare and contrast them.","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Installation & Administration","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Internal Architecture","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data retention","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Integration with the ecosystem","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Pricing","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Kafka vs Kinesis: Installation and administration","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka is an open-source product. That gives you more flexibility in installing, scaling, and operating a Kafka cluster in production. You get to decide the exact Kafka version, the number of brokers, and their hardware specifications based on the workload.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"But this flexibility often comes with a cost.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka is a distributed system made of multiple servers (brokers). Installing and configuring a Kafka cluster to handle a typical production workload might take weeks. Once provisioned, operating a production Kafka cluster requires regular monitoring. You may have to spend on additional hardware to fine-tune the cluster performance to handle spikes in workloads. These factors may result in a high operational cost in terms of billable engineering hours and hardware.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conversely, Kinesis hides many operational complexities from its users by being fully managed. Kinesis only exposes its users to the interfaces that matter the most--APIs for reading and writing data and configurations for securing and scaling Kinesis to handle a production workload. Apart from that, AWS takes over the laborious tasks of hardware provisioning, software installation and patching, monitoring, and other management aspects.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Kafka vs Kinesis: Internal architecture - partitions vs. shards","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Although Kafka and Kinesis are trying to solve the same problem, they do it differently. Both technologies have their architectural differences.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Kafka has partitions","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka organizes its events around topics where all related events are written to the same topic. A topic is further broken down into partitions.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A partition is the smallest unit in a Kafka cluster that stores a subset of events belonging to a topic. Events written to a partition are strictly ordered by their partition key. When Kafka receives an event, the partition key associated with the event is passed into a hash function to determine which partition the event belongs to. Simply put, events with the same partition key will end up in the same partition.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7IzkfWj0SOciX3FeE2cC8W","type":"Asset","createdAt":"2022-02-08T12:30:11.534Z","updatedAt":"2023-01-30T13:47:37.636Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Kafka Topic has many partitions","description":"Kafka Topic Partitions Diagram","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7IzkfWj0SOciX3FeE2cC8W/0ba0582cb425bc5277dcf66184d3ab92/Kafka_Topic_has_many_partitions.png","details":{"size":80638,"image":{"width":3416,"height":2012}},"fileName":"Kafka Topic has many partitions.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Many producers can write to a partition, which can be consumed by many consumers simultaneously. The more partitions you have, the more consumers you can have in parallel. Kafka gives you complete freedom to define any number of topics and partitions based on the cluster specifications.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Kinesis has shards","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kinesis abstracts away many internal details and surfaces only a few key concepts as a managed service. In Kinesis, you have streams, the Kafka equivalent for a topic. All related events are stored in a stream. A stream is further broken down into shards. A shard holds a subset of data records in a stream. Kinesis also uses a partition key to determine the shard a given event belongs to. Like Kafka, events with the same partition key will always end up in the same shard.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Different hashing strategies for message keys","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"However, Kafka and Kinesis use different strategies to hash partition keys. Kafka’s hashing algorithm depends on the number of partitions. If you add a new partition to a topic, Kafka will trigger a rebalance to distribute the partitions to the existing consumers, but the existing data will not be rebalanced. As a result, you will lose the key-based ordering of messages.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conversely, Kinesis hashes message keys on a continuous space regardless of the shard count. Hence, adding or removing shards does not affect the key-based message ordering.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Pull Model: the throughput of a Kinesis shard is bounded","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Unlike a Kafka partition, the throughput of a shard has limits. Each shard can only accept 1,000 records or 1 MB per second (see ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecord.html"},"content":[{"nodeType":"text","value":"PutRecord","marks":[],"data":{}}]},{"nodeType":"text","value":" documentation). If you write to a single shard faster than this rate, you'll get a ProvisionedThroughputExceededException. Each of the shards in a stream supports a read throughput of 2MB per second, which all stream consumers share. You can scale this limit by adding more shards to the stream, incurring additional costs. Comparatively, the throughput of a Kafka partition has no limits.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Push Model: Kinesis supports Enhanced fan-out","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"AWS ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://aws.amazon.com/blogs/aws/kds-enhanced-fanout/"},"content":[{"nodeType":"text","value":"recently announced the Enhanced fan-out feature","marks":[],"data":{}}]},{"nodeType":"text","value":", where each consumer reading from a shard will get a dedicated throughput of 2MB per second. When the fan-out is enabled, records are pushed immediately to the consumer, making it an ideal choice to build real-time, low-latency consumer applications.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conversely, Kafka only supports the traditional read model where consumers are supposed to pull data from partitions.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Kafka vs Kinesis: Data retention period","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We define the data retention period of a streaming platform as the period certain data records are accessible after they are added to the stream.\nBy default, Kafka retains data records for up to seven days. But that is configurable. Often, users tune their Kafka cluster to store events based on several factors such as record size, event arrival rate, and the business use case.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"However, higher retention periods demand more disk space. You can keep on provisioning more local storage for a Kafka cluster, but at some point, it becomes unmanageable. The recommendation is not to use Kafka as eternal storage as it can lead to difficulties in recovering from failures, increased time in maintenance and data migration, etc.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://cwiki.apache.org/confluence/display/KAFKA/KIP-405%3A+Kafka+Tiered+Storage"},"content":[{"nodeType":"text","value":"KIP-405","marks":[],"data":{}}]},{"nodeType":"text","value":" is a proposal to introduce tiered storage to Kafka. That allows keeping the latest data for a short period (e.g., 48 hours) in the local storage and flushing out the older data into cheap secondary storage like S3 or HDFS. Use cases like hydrating a new database, training a new machine learning model, and testing a new version of a consumer can always reach the secondary storage to access the older data.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Stream retention period on Kinesis is usually set to a default of 24 hours after creation. But Kinesis allows users to increase the retention period up to 365 days. Additional costs will incur if they want to retain data for more than 24 hours.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Kafka vs Kinesis: SDKs and ecosystem integration","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"An event streaming platform is useless without having a well-documented SDK for developers to read and write events into it.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka officially provides two types of SDK for Java developers. First, Kafka offers low-level producer and consumer APIs targeting straightforward event production and consumer use cases. Secondly, Kafka provides the ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/documentation/streams/"},"content":[{"nodeType":"text","value":"Kafka Streams API","marks":[],"data":{}}]},{"nodeType":"text","value":" to implement more complex stream processing use cases such as event-driven microservices, fraud detection, and real-time analytics.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apart from Java, Kafka has ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://cwiki.apache.org/confluence/display/KAFKA/Clients"},"content":[{"nodeType":"text","value":"community-maintained SDKs for other languages","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/documentation/#connect"},"content":[{"nodeType":"text","value":"Kafka Connect","marks":[],"data":{}}]},{"nodeType":"text","value":" is a language-neutral tool for scalably and reliably streaming data between Apache Kafka and other systems. Kafka Connect comes with “connectors” to different ecosystem components that enable moving a large data collection into and out of Kafka. For example, events received into a Kafka topic can be sent to Elasticsearch by routing through Kafka Connect with the Elasticsearch connector.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1v7OBTO0mZ4GRjEcD4C1LI","type":"Asset","createdAt":"2022-02-08T12:34:40.120Z","updatedAt":"2023-01-30T13:48:30.853Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Kafka Connect Architecture","description":"Diagram showing basic Kafka Connect architecture","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1v7OBTO0mZ4GRjEcD4C1LI/eeefa4ae8acb15967a0e7ad058d754d7/Kafka_Connect_Architecture.png","details":{"size":37984,"image":{"width":2882,"height":648}},"fileName":"Kafka Connect Architecture.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Amazon SDKs for Go, Java, JavaScript, .NET, Node.js, PHP, Python, and Ruby supports Kinesis Data Streams. In addition, the ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.aws.amazon.com/streams/latest/dev/shared-throughput-kcl-consumers.html"},"content":[{"nodeType":"text","value":"Kinesis Client Library (KCL)","marks":[],"data":{}}]},{"nodeType":"text","value":" provides an easy-to-use programming model for processing data, and the users can get started quickly with Kinesis Data Streams in Java, Node.js, .NET, Python, and Ruby.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Events written to a Kinesis stream can be taken out to other AWS services via ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html"},"content":[{"nodeType":"text","value":"AWS Kinesis Data Firehose","marks":[],"data":{}}]},{"nodeType":"text","value":"; the Kafka Connect equivalent connects Kinesis to other ecosystem products like S3, Redshift, and Splunk. Kinesis also tightly integrates with ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.aws.amazon.com/kinesisanalytics/latest/dev/what-is.html"},"content":[{"nodeType":"text","value":"Kinesis Data Analytics","marks":[],"data":{}}]},{"nodeType":"text","value":" (KDA), allowing developers to build stream processing applications on top of the events flowing from Kinesis.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In fact, KDA is Apache Flink as a managed service. It provides an alternative to Kafka Streams. You can also use KDA against a Kafka cluster to deploy your Flink applications.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2orjyJYgZ0pAHX0J8GMewx","type":"Asset","createdAt":"2022-02-08T12:37:09.425Z","updatedAt":"2023-01-30T13:49:04.536Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Amazon Kinesis Data Firehose Architecture","description":"Overview of Amazon Kinesis Firehose Architecture","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2orjyJYgZ0pAHX0J8GMewx/b43ddb5fab4aed1a019257fc4888f531/Kinesis_Data_Firehose_Architecture.png","details":{"size":88309,"image":{"width":3416,"height":1370}},"fileName":"Kinesis Data Firehose Architecture.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Kafka vs Kinesis: Pricing","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka is an open-source product. Meaning it incurs zero upfront cost to get started. However, as we discussed earlier, operating a production Kafka cluster incurs costs in terms of hardware and labor.\nAmazon’s model for Kinesis is pay-as-you-go. It works on the principle that there are no upfront costs for setting up; the amount to be paid depends on the services rendered. The pricing is calculated in terms of shard hours, payload units, or data retention period.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You can find more information on Kinesis pricing from ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://aws.amazon.com/kinesis/data-streams/pricing/"},"content":[{"nodeType":"text","value":"here","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Our discussion so far can be summarised as follows:","marks":[],"data":{}}]},{"nodeType":"table","data":{},"content":[{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-header-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Feature","marks":[],"data":{}}]}]},{"nodeType":"table-header-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka","marks":[],"data":{}}]}]},{"nodeType":"table-header-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kinesis","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Licensing","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Free and open-source","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Commercial","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Hosting","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Primarily on-premises, but managed services like Confluent, AWS MSK, and Aiven also exist.","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Only on the AWS cloud. AWS takes care of the management.","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Logical records container","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Topic","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Stream","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Physical records container","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Partition","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Shard","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Key hashing strategy","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Depends on the number of partitions (breaks key-based ordering if adding partitions)","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Independent of shards (doesn’t break key-based ordering if adding shards)","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Throughput per partition/shard","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Unlimited. Only governed by cluster’s resources","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Both read and write throughput is limited per shard. Scale-out by adding more shards. More throughput for consumers if using enhanced fan-out","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Message consumption strategy","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Consumers pull data from partitions","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Consumer-driven pull and enhanced fan-out where messages are being pushed to consumers.","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data retention","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"One week by default. But configurable. The tiered storage capability is working progress.","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"24 hours by default. But can be extended up to 365 days which would incur a cost.","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"SDK support","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Official Java SDK and community maintained SDKs for other languages.","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"SDKs are available in many languages and integrate well with AWS CLI.","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ecosystem integration options","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka Connect","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kinesis Data Firehose","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Stream processing engine integration","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka Streams, ksqlDB","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kinesis Data Analytics (managed Apache Flink)","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Operational complexity","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Takes weeks to set up a production cluster yourself, hours if using managed solutions","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Provisions in hours","marks":[],"data":{}}]}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The technology space we live in today is full of choices, making it challenging to come up with a clear answer to many technical decisions. The same applies when choosing either Kafka or Kinesis as an event streaming platform. There’s no single correct answer. It always depends.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If you come from a background where the cloud is no option, you have access to engineering talent experienced in distributed systems, DevOps, and JVM languages; Kafka might be a good fit for your organization.\nOn the other hand, if you are starting small or have been established on AWS for a while, Kinesis provides you with a smooth transition to event streaming. It takes away the operational burden and lets you focus only on the business problem while giving you the best value for your investment.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We hope this article helped you pick the right technology based on the engineering culture, budgetary constraints, and how critical the role of event streaming plays within your organization.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"And if you choose Apache Kafka for your data event streaming development, make sure to check out Conduktor. Our all-in-one GUI works whether your Kafka is on-prem or you're using a managed service or cloud solution like Amazon MSK, Aiven, Confluent, or Upstash. Ultimately, Conduktor will help to bypass some of the speedbumps, increase productivity, reduce costs and ultimately, accelerate project delivery.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Download Conduktor now.","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},"body":"Gone are the days when organizations used to make decisions based on emotions and experience. With the inception of modern data analytics and machine learning technologies, organizations are transitioning into data-driven, real-time business decision-making.\n\nTechnologies for collecting, storing, and processing business events in real-time are getting more critical than ever. Apache Kafka and AWS Kinesis are two event streaming platforms that enable ingesting a large number of events each second and storing them durably until they are analyzed. \n\nThis article compares Kafka and Kinesis across five dimensions. Hopefully, it will provide you with a useful reference for picking between them in the future.\n\n## Apache Kafka - An overview\n\n[Kafka](https://kafka.apache.org/ \"Apache Kafka Website\") was originally developed at LinkedIn as a publish-subscribe system. But, it has evolved into a distributed, fault-tolerant, and scalable event streaming platform. \n\nKafka is designed to operate as a distributed system that could span multiple data centers. A Kafka cluster is a collection of brokers who organize events into topics and store them durably for a configurable amount of time. \n\nKafka provides a set of APIs for reading and writing data. Applications such as web applications, IoT devices, and Microservices could use the [Producer API](https://kafka.apache.org/documentation/#producerapi \"Apache Kafka Producer API\") to write events into a Kafka topic. Consumer applications like stream processors and analytics databases subscribe to a topic and read events using [Consumer API](https://kafka.apache.org/documentation/#consumerapi \"Apache Kafka Consumer API\"). Multiple producers can simultaneously produce events to a topic while many consumers already consume from the same topic. That leads to virtually unlimited scalability.\n![Apache Kafka Architecture](//images.ctfassets.net/lm7q36eaz111/mSYc3d9f4n5lKQ8W4GC0g/d01b03f76f0f113896ece90b490bba83/Apache_Kafka_Architecture.png)\n*Figure 01 - Apache Kafka architecture*\n
\n\nKafka was donated to the Apache Software Foundation (ASF) to become an open-source project. That means you can download, install, and manage an Apache Kafka cluster free of charge. Also, Kafka is available as a managed service from vendors such as [Confluent](http://confluent.io \"Confluent\"), [AWS MSK](https://aws.amazon.com/msk/ \"Amazon MSK\"), and [Aiven](https://aiven.io/ \"Aiven\").\n\n## AWS Kinesis - An overview\n\nKinesis is an event streaming platform based on the publish-subscribe principle. Kinesis is offered as a managed service by AWS.\n\nKinesis is the umbrella term used for four different services--Kinesis Data Streams, Kinesis Data Firehose, Kinesis Video Streams, and Kinesis Data Analytics. While each service serves a specific purpose, we will only consider Kinesis Data Streams for the comparison as it provides a foundation for the rest of the services. We will refer to Kinesis Data Streams as Kinesis for the sake of simplicity.\n\nKinesis organizes its data records into [shards](https://docs.aws.amazon.com/streams/latest/dev/key-concepts.html#shard \"Amazon Kinesis Shard\"). Like Kafka, Kinesis also provides its users with APIs for producing and consuming events into a shard. Those APIs are available in multiple programming languages and tightly integrated with AWS SDKs and the CLI.\n![Amazon Kinesis Architecture](//images.ctfassets.net/lm7q36eaz111/4DzhpjqRuexCYlFAXXEod3/e73607eae7a1e712cb7e4c028b247d46/Amazon_Kinesis_Architecture.png)\n*Figure 02 - AWS Kinesis architecture*\n
\n\n## Kafka vs. Kinesis - Dimensions for the comparison\n\nBoth Kafka and Kinesis are prominent technologies in the event streaming space. A decision to choose either of them should be made rationally. Hence, we came up with the following dimensions to compare and contrast them.\n- Installation & Administration\n- Internal Architecture\n- Data retention\n- Integration with the ecosystem\n- Pricing\n\n## Kafka vs Kinesis: Installation and administration\n\nApache Kafka is an open-source product. That gives you more flexibility in installing, scaling, and operating a Kafka cluster in production. You get to decide the exact Kafka version, the number of brokers, and their hardware specifications based on the workload.\n\nBut this flexibility often comes with a cost.\n\nKafka is a distributed system made of multiple servers (brokers). Installing and configuring a Kafka cluster to handle a typical production workload might take weeks. Once provisioned, operating a production Kafka cluster requires regular monitoring. You may have to spend on additional hardware to fine-tune the cluster performance to handle spikes in workloads. These factors may result in a high operational cost in terms of billable engineering hours and hardware.\n\nConversely, Kinesis hides many operational complexities from its users by being fully managed. Kinesis only exposes its users to the interfaces that matter the most--APIs for reading and writing data and configurations for securing and scaling Kinesis to handle a production workload. Apart from that, AWS takes over the laborious tasks of hardware provisioning, software installation and patching, monitoring, and other management aspects. \n\n## Kafka vs Kinesis: Internal architecture - partitions vs. shards\n\nAlthough Kafka and Kinesis are trying to solve the same problem, they do it differently. Both technologies have their architectural differences. \n\n### Kafka has partitions\nKafka organizes its events around topics where all related events are written to the same topic. A topic is further broken down into partitions. \n\nA partition is the smallest unit in a Kafka cluster that stores a subset of events belonging to a topic. Events written to a partition are strictly ordered by their partition key. When Kafka receives an event, the partition key associated with the event is passed into a hash function to determine which partition the event belongs to. Simply put, events with the same partition key will end up in the same partition. \n![Kafka Topic has many partitions](//images.ctfassets.net/lm7q36eaz111/7IzkfWj0SOciX3FeE2cC8W/0ba0582cb425bc5277dcf66184d3ab92/Kafka_Topic_has_many_partitions.png)\n*Figure 03 - Kafka topic has many partitions*\n
\n\nMany producers can write to a partition, which can be consumed by many consumers simultaneously. The more partitions you have, the more consumers you can have in parallel. Kafka gives you complete freedom to define any number of topics and partitions based on the cluster specifications.\n\n### Kinesis has shards\nKinesis abstracts away many internal details and surfaces only a few key concepts as a managed service. In Kinesis, you have streams, the Kafka equivalent for a topic. All related events are stored in a stream. A stream is further broken down into shards. A shard holds a subset of data records in a stream. Kinesis also uses a partition key to determine the shard a given event belongs to. Like Kafka, events with the same partition key will always end up in the same shard. \n\n### Different hashing strategies for message keys\nHowever, Kafka and Kinesis use different strategies to hash partition keys. Kafka’s hashing algorithm depends on the number of partitions. If you add a new partition to a topic, Kafka will trigger a rebalance to distribute the partitions to the existing consumers, but the existing data will not be rebalanced. As a result, you will lose the key-based ordering of messages.\n\nConversely, Kinesis hashes message keys on a continuous space regardless of the shard count. Hence, adding or removing shards does not affect the key-based message ordering.\n\n### Pull Model: the throughput of a Kinesis shard is bounded\nUnlike a Kafka partition, the throughput of a shard has limits. Each shard can only accept 1,000 records or 1 MB per second (see [PutRecord](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecord.html \"PutRecord Documentation\") documentation). If you write to a single shard faster than this rate, you'll get a ProvisionedThroughputExceededException. Each of the shards in a stream supports a read throughput of 2MB per second, which all stream consumers share. You can scale this limit by adding more shards to the stream, incurring additional costs. Comparatively, the throughput of a Kafka partition has no limits.\n\n### Push Model: Kinesis supports Enhanced fan-out\nAWS [recently announced the Enhanced fan-out feature](https://aws.amazon.com/blogs/aws/kds-enhanced-fanout/ \"KDS enhanced fan out feature on Kinesis\"), where each consumer reading from a shard will get a dedicated throughput of 2MB per second. When the fan-out is enabled, records are pushed immediately to the consumer, making it an ideal choice to build real-time, low-latency consumer applications.\n\nConversely, Kafka only supports the traditional read model where consumers are supposed to pull data from partitions.\n\n## Kafka vs Kinesis: Data retention period\n\nWe define the data retention period of a streaming platform as the period certain data records are accessible after they are added to the stream. \nBy default, Kafka retains data records for up to seven days. But that is configurable. Often, users tune their Kafka cluster to store events based on several factors such as record size, event arrival rate, and the business use case.\n\nHowever, higher retention periods demand more disk space. You can keep on provisioning more local storage for a Kafka cluster, but at some point, it becomes unmanageable. The recommendation is not to use Kafka as eternal storage as it can lead to difficulties in recovering from failures, increased time in maintenance and data migration, etc. \n\n[KIP-405](https://cwiki.apache.org/confluence/display/KAFKA/KIP-405%3A+Kafka+Tiered+Storage \"KIP-405 Proposal to introduce tiered storage to Kafka\") is a proposal to introduce tiered storage to Kafka. That allows keeping the latest data for a short period (e.g., 48 hours) in the local storage and flushing out the older data into cheap secondary storage like S3 or HDFS. Use cases like hydrating a new database, training a new machine learning model, and testing a new version of a consumer can always reach the secondary storage to access the older data.\n\nStream retention period on Kinesis is usually set to a default of 24 hours after creation. But Kinesis allows users to increase the retention period up to 365 days. Additional costs will incur if they want to retain data for more than 24 hours.\n\n## Kafka vs Kinesis: SDKs and ecosystem integration\n\nAn event streaming platform is useless without having a well-documented SDK for developers to read and write events into it.\n\nKafka officially provides two types of SDK for Java developers. First, Kafka offers low-level producer and consumer APIs targeting straightforward event production and consumer use cases. Secondly, Kafka provides the [Kafka Streams API](https://kafka.apache.org/documentation/streams/ \"Kafka Streams API\") to implement more complex stream processing use cases such as event-driven microservices, fraud detection, and real-time analytics. \n\nApart from Java, Kafka has [community-maintained SDKs for other languages](https://cwiki.apache.org/confluence/display/KAFKA/Clients \"Apache Kafka SDKs for other languages\").\n\n[Kafka Connect](https://kafka.apache.org/documentation/#connect \"Kafka Connect Docs\") is a language-neutral tool for scalably and reliably streaming data between Apache Kafka and other systems. Kafka Connect comes with “connectors” to different ecosystem components that enable moving a large data collection into and out of Kafka. For example, events received into a Kafka topic can be sent to Elasticsearch by routing through Kafka Connect with the Elasticsearch connector.\n![Kafka Connect Architecture](//images.ctfassets.net/lm7q36eaz111/1v7OBTO0mZ4GRjEcD4C1LI/eeefa4ae8acb15967a0e7ad058d754d7/Kafka_Connect_Architecture.png)\n*Figure 04 - Kafka Connect architecture*\n
\n\nAmazon SDKs for Go, Java, JavaScript, .NET, Node.js, PHP, Python, and Ruby supports Kinesis Data Streams. In addition, the [Kinesis Client Library (KCL)](https://docs.aws.amazon.com/streams/latest/dev/shared-throughput-kcl-consumers.html \"Amazon KCL\") provides an easy-to-use programming model for processing data, and the users can get started quickly with Kinesis Data Streams in Java, Node.js, .NET, Python, and Ruby.\n\nEvents written to a Kinesis stream can be taken out to other AWS services via [AWS Kinesis Data Firehose](https://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html \"AWS Kinesis Firehouse\"); the Kafka Connect equivalent connects Kinesis to other ecosystem products like S3, Redshift, and Splunk. Kinesis also tightly integrates with [Kinesis Data Analytics](https://docs.aws.amazon.com/kinesisanalytics/latest/dev/what-is.html \"Kinesis Data Analytics\") (KDA), allowing developers to build stream processing applications on top of the events flowing from Kinesis.\n\nIn fact, KDA is Apache Flink as a managed service. It provides an alternative to Kafka Streams. You can also use KDA against a Kafka cluster to deploy your Flink applications.\n![Amazon Kinesis Data Firehose Architecture](//images.ctfassets.net/lm7q36eaz111/2orjyJYgZ0pAHX0J8GMewx/b43ddb5fab4aed1a019257fc4888f531/Kinesis_Data_Firehose_Architecture.png)\n*Figure 05 - Kinesis Data Firehose architecture*\n
\n\n## Kafka vs Kinesis: Pricing\n\nKafka is an open-source product. Meaning it incurs zero upfront cost to get started. However, as we discussed earlier, operating a production Kafka cluster incurs costs in terms of hardware and labor.\nAmazon’s model for Kinesis is pay-as-you-go. It works on the principle that there are no upfront costs for setting up; the amount to be paid depends on the services rendered. The pricing is calculated in terms of shard hours, payload units, or data retention period.\n\nYou can find more information on Kinesis pricing from [here](https://aws.amazon.com/kinesis/data-streams/pricing/ \"Amazon Kinesis Pricing\"). \n\n## Conclusion\n\nOur discussion so far can be summarised as follows:\n\n| Feature | Kafka | Kinesis |\n| ------------------------------------ | ------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------- |\n| Licensing | Free and open-source | Commercial |\n| Hosting | Primarily on-premises, but managed services like Confluent, AWS MSK, and Aiven also exist. | Only on the AWS cloud. AWS takes care of the management. |\n| Logical records container | Topic | Stream |\n| Physical records container | Partition | Shard |\n| Key hashing strategy | Depends on the number of partitions (breaks key-based ordering if adding partitions) | Independent of shards (doesn’t break key-based ordering if adding shards) |\n| Throughput per partition/shard | Unlimited. Only governed by cluster’s resources | Both read and write throughput is limited per shard. Scale-out by adding more shards. More throughput for consumers if using enhanced fan-out |\n| Message consumption strategy | Consumers pull data from partitions | Consumer-driven pull and enhanced fan-out where messages are being pushed to consumers. |\n| Data retention | One week by default. But configurable. The tiered storage capability is working progress. | 24 hours by default. But can be extended up to 365 days which would incur a cost. |\n| SDK support | Official Java SDK and community maintained SDKs for other languages. | SDKs are available in many languages and integrate well with AWS CLI. |\n| Ecosystem integration options | Kafka Connect | Kinesis Data Firehose |\n| Stream processing engine integration | Kafka Streams, ksqlDB | Kinesis Data Analytics (managed Apache Flink) |\n| Operational complexity | Takes weeks to set up a production cluster yourself, hours if using managed solutions | Provisions in hours |\n\nThe technology space we live in today is full of choices, making it challenging to come up with a clear answer to many technical decisions. The same applies when choosing either Kafka or Kinesis as an event streaming platform. There’s no single correct answer. It always depends.\n\nIf you come from a background where the cloud is no option, you have access to engineering talent experienced in distributed systems, DevOps, and JVM languages; Kafka might be a good fit for your organization.\nOn the other hand, if you are starting small or have been established on AWS for a while, Kinesis provides you with a smooth transition to event streaming. It takes away the operational burden and lets you focus only on the business problem while giving you the best value for your investment.\n\nWe hope this article helped you pick the right technology based on the engineering culture, budgetary constraints, and how critical the role of event streaming plays within your organization.\n\nAnd if you choose Apache Kafka for your data event streaming development, make sure to check out Conduktor. Our all-in-one GUI works whether your Kafka is on-prem or you're using a managed service or cloud solution like Amazon MSK, Aiven, Confluent, or Upstash. Ultimately, Conduktor will help to bypass some of the speedbumps, increase productivity, reduce costs and ultimately, accelerate project delivery. \n\n[Download Conduktor now.](https://conduktor.io/download/ \"Download Conduktor\")","author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2SX6zpOndgjWDVop0j59Uz","type":"Entry","createdAt":"2021-10-08T17:41:29.513Z","updatedAt":"2022-09-27T10:40:02.619Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":6,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Stéphane","surname":"Maarek","title":"Chief Evangelist","twitter":"https://twitter.com/StephaneMaarek","github":"https://github.com/simplesteph","linkedIn":"https://www.linkedin.com/in/stephanemaarek/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7y6tdCR7qU2yEWMJmQDS23","type":"Asset","createdAt":"2021-10-08T17:40:54.512Z","updatedAt":"2022-09-25T07:11:34.678Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Stephane Maarek","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7y6tdCR7qU2yEWMJmQDS23/523069a652132097fc9825d7831c0790/stephane-m.png","details":{"size":47223,"image":{"width":192,"height":192}},"fileName":"stephane-m.png","contentType":"image/png"}}}}},"publishDate":"2022-02-08T17:00+00:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2ZAVT9lSo8mSHCedaIgOKg","type":"Entry","createdAt":"2021-11-24T15:55:21.995Z","updatedAt":"2022-09-25T13:49:44.587Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":5,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Ensuring Maximum Security with Conduktor Desktop v2.20.1","description":"Our latest release updates a number of dependencies that we identified as having potential security weaknesses.","noIndex":false}},"slug":"ensuring-maximum-security-conduktor-desktop-v2.20.1","title":"Ensuring Maximum Security with Conduktor Desktop v2.20.1","description":"Our latest release updates a number of dependencies to ensure maximum security for Conduktor Desktop users.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"sNPxksP7DPWqhhVFsqW8M","type":"Asset","createdAt":"2021-12-17T10:55:23.973Z","updatedAt":"2022-09-25T10:35:59.757Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Conduktor Security Update","description":"Conduktor Desktop security update","file":{"url":"//images.ctfassets.net/lm7q36eaz111/sNPxksP7DPWqhhVFsqW8M/e28e6c286aa62c4993f46f7b25c2b30e/Conduktor_Desktop_-_Enhanced_Security.jpg","details":{"size":141319,"image":{"width":2200,"height":1467}},"fileName":"Conduktor Desktop - Enhanced Security.jpg","contentType":"image/jpeg"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"While Conduktor Desktop was not affected by the latest ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-44228"},"content":[{"nodeType":"text","value":"CVE","marks":[],"data":{}}]},{"nodeType":"text","value":" discovered in Log4J v2, this event helped the JVM community notice some similar, yet less severe vulnerabilities, present in Logback, the SLF4J implementation used at Conduktor and shipped with Conduktor Desktop.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Background","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We became aware of this vulnerability, ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-44228"},"content":[{"nodeType":"text","value":"CVE-2021-44228","marks":[],"data":{}}]},{"nodeType":"text","value":", when it was announced late last week and immediately conducted a comprehensive analysis of our build to identify any resources using the Apache Log4J logging library.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This analysis showed that; ","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We were not vulnerable to this CVE (confirmed by an analysis made with ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://snyk.io/"},"content":[{"nodeType":"text","value":"Snyk","marks":[],"data":{}}]},{"nodeType":"text","value":" on version v2.19.3 of Conduktor Desktop)","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"None of our dependencies were dependent on Log4j v2.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor Desktop and the libraries used in Conduktor Desktop were not, and are not, using Log4j v2.","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Our analysis did, however, identify some potential issues with Logback v1.2.7 and Log4J v1. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As explained by the Logback team ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"http://logback.qos.ch/news.html"},"content":[{"nodeType":"text","value":"here","marks":[],"data":{}}]},{"nodeType":"text","value":", the versions before v1.2.8 of Logback were affected by some security issues fixed in v1.2.8 and later.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Prior versions of Conduktor Desktop are shipped with Logback v1.2.7 which contains some security issues.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Logback also released a new v1.2.9 version yesterday (2021.12.16), hardening its code and preventing new potential security issues.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The latest version of Conduktor Desktop, v2.20.1, ships with Logback v1.2.9 to ensure maximum security for our clients.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Another potential issue affecting Conduktor Desktop prior to v2.20.0 was that some dependencies were using Log4J v1, which is not affected by the latest Log4J v2 CVE, but is unmaintained and may contain some security issues too.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To fix this issue, all the dependencies on Log4J have been filtered out of Conduktor Desktop.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":" To conclude,","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor Desktop was not affected by the latest CVE discovered in Log4J v2,","marks":[{"type":"bold"}],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"All the dependencies on Log4J v1 have been removed in the v2.20.0+","marks":[{"type":"bold"}],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The latest version of Logback is shipped in our latest release, v2.20.1.","marks":[{"type":"bold"}],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Update Now","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"At Conduktor, safeguarding the security of our users’ data is at the heart of everything we do. We work hard to improve the app with each new version and as such, we always recommend updating to the latest version of Conduktor to ensure maximum security.","marks":[],"data":{}}]},{"nodeType":"heading-4","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Download Conduktor Desktop 2.20.1 now!","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5au3vTjeNc7Ctl5QELPO4O","type":"Entry","createdAt":"2021-10-08T18:56:16.912Z","updatedAt":"2022-09-25T14:05:25.231Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Jules","title":"Software Engineer","twitter":"https://twitter.com/guizmaii","github":"https://github.com/guizmaii","linkedIn":"https://www.linkedin.com/in/jules-ivanic/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"xAViG6Oc1Cjy9LhsG7dZ3","type":"Asset","createdAt":"2022-09-25T14:05:22.158Z","updatedAt":"2022-09-25T14:05:22.158Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"jules","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/xAViG6Oc1Cjy9LhsG7dZ3/986f217ee81457b0332c54576efed311/jules.png","details":{"size":50487,"image":{"width":192,"height":192}},"fileName":"jules.png","contentType":"image/png"}}}}},"publishDate":"2021-12-17T11:00+00:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5WOxUXPoTFJeIRvp2Rukkg","type":"Entry","createdAt":"2021-11-02T16:41:08.774Z","updatedAt":"2022-09-25T13:48:12.252Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Real-time data visualisation","description":"See how Conduktor helps extracting and visualization interesting insights out of your data.","noIndex":false}},"slug":"real-time-data-visualization","title":"Real-time data visualization","description":"See how Conduktor helps extracting and visualization interesting insights out of your data.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5kv1HrGdEPTEO5A2BGh6jy","type":"Asset","createdAt":"2021-11-09T14:16:19.272Z","updatedAt":"2022-09-25T07:23:34.776Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Real time data visualization","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5kv1HrGdEPTEO5A2BGh6jy/01fc3cd0c2b1f1dba34eeb335ca04b14/blog-conduktor03.png","details":{"size":78269,"image":{"width":640,"height":400}},"fileName":"blog-conduktor03.png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Big data movement has led to an exponential growth in data captured by every business. The data stretch from high-level information like customer surveys, market movements, etc, to finer product level information like user trails, feature clicks, and metrics (business and application).","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The business requirements around data are served by many open-source data streaming platforms like Apache Kafka. Generating data is only the start to do Data-Driven Decision Making (DDDM).","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Data-driven decision-making","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data-driven decision-making (DDDM) is defined as making decisions based on hard data as opposed to intuition, observation, or guesswork. The value of data-driven decisions is dependent on the quality of the data and its analysis and interpretation.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"A need for Data Visualization","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data are a powerful asset when there is an opportunity to access them and visualize the value they capture.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data visualization solutions provide ways to access and filter the most relevant data and show it in different formats. Unlike a data-streaming platform, used by developers and ops, the data visualization platform is used by Business Analysts and Stakeholders. It must offer features that are more suited for non-technical users, intuitive, fast, and beautiful!","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Apache Kafka as a leading data platform","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.confluent.io/kafka/introduction.html"},"content":[{"nodeType":"text","value":"https://docs.confluent.io/5.5.1/kafka/introduction.html","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka has been the cornerstone of all the data streaming transformations which yield real-time data collection, processing, storage, and analysis.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"It has been adopted by over 60% of Fortune 100 companies. The open-source software is often an underlying component in most streaming architectures. There is a high probability that your business has adopted Kafka or is looking to use it, to help to transform their IT and lead their digital data transformation.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To make the best use of data flowing through Apache Kafka, we need a data visualization solution with proper integration to Apache Kafka and visualize data in almost-real-time, not tomorrow or a week.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Also, due to the vast range of data formats these days (more or less specialized), a data visualization platform must be able to understand data published in different formats (CSV, XML, JSON, Apache Avro, Google Protobuf, MessagePack, etc.) to be efficient and avoid useless data-conversions preparation jobs.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Platform Agnostic","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka (or any other data-streaming solution) is often deployed across diverse infrastructure platforms like cloud, on-premise VMs, containers, etc. A successful data visualization solution must support connectivity to all of the various deployments while enforcing the required security and governance.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Business First","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The solution we're looking for must be business-users-focused rather than developer-focused.\nThe platform must be easy to operate by Data Analysts and stakeholders without the need of any developer.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A data streaming platform typically processes a bunch of events at any given instant. After a bit of tuning and with the proper infrastructure, Apache Kafka can quickly process 2 million records. It should be \"easy\" and intuitive to display aggregated values in real-time, to provide instant insights, and be able to act upon them quickly. Also, looking at the whole history at a glance (day, week, year) is a must to see the evolution and trends.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conduktor: an intuitive interface to peek into your data","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We provide a desktop application with a friendly user interface to work with Apache Kafka and its extensions (Schema Registry, Kafka Connect, Kafka Streams, ksqlDB, etc.).","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor enables you to look into the data published into your Apache Kafka topics. It is compatible with most data formats: from the basics to the most complex.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You can search, analyze, and export data published in real-time or from the past and analyze them. It's often a requirement for developers, QA, and Data Analysts.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Working with Conduktor","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let's walk through how to work with Conduktor. We need to make sure to have the following:","marks":[],"data":{}}]},{"nodeType":"ordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Access to a running Apache Kafka cluster.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Download","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":" and install Conduktor on your own machine","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Signup/Login into Conduktor, you will be able to synchronize your clusters with your team or create new clusters:","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Connect Conduktor to your Apache Kafka cluster:","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Connectivity","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor supports on-premise installations, Kubernetes custom installations, Cloud-managed installations, as well as service providers like Confluent Cloud, Aiven, CloudKarafka, you name it.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You can validate connection details before saving them and iterate easily in case of trouble. You can connect Conduktor to other features outside of our scope here: SSH, Schema Registry, Kafka Connect, ksqlDB, etc. These are not immediately applicable to business stakeholders but are relevant for Operational needs.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Once connected, you've presented a dashboard with an overview of your Apache Kafka cluster, and important metrics to watch for. Conduktor will warn you if it detects anything wrong (misconfiguration, topics failures, Kafka Streams applications down...).","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Data Exploration","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka has its data organized in topics. It's quite normal to have hundreds of them in a small business, and thousands in larger businesses. The topics are named according to the data they serve and their level of security, privacy, business units, etc.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A few examples of classic analytics data published into Apache Kafka are: user signups, notifications, and product feature clicks.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You can explore the data published on any of the topics by clicking the search icon associated with each Topic. In the above cluster, let's look into the notifications topic.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Data Lookup and Selection","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor provides a powerful data lookup capability.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"It offers flexibility to lookup data across various timeframes like current data, last hour, yesterday, since the beginning, etc. It provides support for various data formats like JSON, Apache Avro (with many variations), Protobuf, JSON-Schema, and binary data. Each topic can have a different format due to specializations, the programming language used, and developer experience.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Here, a topic using the JSON format:","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You can start building insights by filtering data based on different criteria: similarity, equality, containment, specific field, and regular expressions. This is available for data and metadata of the records.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Here, we're lookup for all the \"email\" notifications types only:","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Data Views","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor provides multiple views to see the data and will keep expanding to provide more perspectives of the same data.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"By default, the view is clear and simple to quickly see the data at a glance. Conduktor also provides a tabular view to display data alongside its other attributes: key, value, timestamp, and headers (deconstructed). This allows for customization (column selection) to reduce data noise by remaining focused on the useful bits of data.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"It's also possible to \"project\" the data to extract only what's necessary.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Here, we focus on the field notification.name by using a \"Field Selection\":","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data features are essential (aggregations in real-time, keeping the latest occurrence by key only...), and we keep adding more and more (charts, histograms, pivot tables)! Feel free to tell us what you'd like to see.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Offline Analysis","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Once we have captured some data, we can export them for offline analysis and share them internally.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor can export data in Excel-supported CSV format or JSON for developers. It's possible to export thousands of records in real-time, or just a small slice. It will allow us to build visualization like this:","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Next steps","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Try Conduktor for free","marks":[],"data":{}}]},{"nodeType":"text","value":" and let us assist you in building your Data-Driven Business.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Thank you.","marks":[],"data":{}}]}]},"body":"The Big data movement has led to an exponential growth in data captured by every business. The data stretch from high-level information like customer surveys, market movements, etc, to finer product level information like user trails, feature clicks, and metrics (business and application).\n\nThe business requirements around data are served by many open-source data streaming platforms like Apache Kafka. Generating data is only the start to do Data-Driven Decision Making (DDDM).\n\n![Data-Driven Decision Making](//images.contentful.com/lm7q36eaz111/6R2DIzDaJ2ay7Ts5DfwxUY/994f9dc769f3e5e6dd15f48885c822e8/dddm.png)\n\n## Data-driven decision-making\n\nData-driven decision-making (DDDM) is defined as making decisions based on hard data as opposed to intuition, observation, or guesswork. The value of data-driven decisions is dependent on the quality of the data and its analysis and interpretation.\n\n## A need for Data Visualization\n\nData are a powerful asset when there is an opportunity to access them and visualize the value they capture.\n\nData visualization solutions provide ways to access and filter the most relevant data and show it in different formats. Unlike a data-streaming platform, used by developers and ops, the data visualization platform is used by Business Analysts and Stakeholders. It must offer features that are more suited for non-technical users, intuitive, fast, and beautiful!\n\n## Apache Kafka as a leading data platform\n\n![Apache Kafka - https://docs.confluent.io/5.5.1/kafka/introduction.html](//images.contentful.com/lm7q36eaz111/7uBX7CshdSmM3tZqJMzaJ1/6a6099e1728fb5f11c0c587675b67a7a/kafka.png)\n[https://docs.confluent.io/5.5.1/kafka/introduction.html](https://docs.confluent.io/5.5.1/kafka/introduction.html)\n\nApache Kafka has been the cornerstone of all the data streaming transformations which yield real-time data collection, processing, storage, and analysis.\n\nIt has been adopted by over 60% of Fortune 100 companies. The open-source software is often an underlying component in most streaming architectures. There is a high probability that your business has adopted Kafka or is looking to use it, to help to transform their IT and lead their digital data transformation.\n\nTo make the best use of data flowing through Apache Kafka, we need a data visualization solution with proper integration to Apache Kafka and visualize data in almost-real-time, not tomorrow or a week.\n\nAlso, due to the vast range of data formats these days (more or less specialized), a data visualization platform must be able to understand data published in different formats (CSV, XML, JSON, Apache Avro, Google Protobuf, MessagePack, etc.) to be efficient and avoid useless data-conversions preparation jobs.\n\n## Platform Agnostic\n\nApache Kafka (or any other data-streaming solution) is often deployed across diverse infrastructure platforms like cloud, on-premise VMs, containers, etc. A successful data visualization solution must support connectivity to all of the various deployments while enforcing the required security and governance.\n\n![Deploying](//images.contentful.com/lm7q36eaz111/29DkHKg5nvcUyCyox34C5y/d74488a1bfaef3e7bc0c510d86670356/deploy.png)\n\n## Business First\n\nThe solution we're looking for must be business-users-focused rather than developer-focused.\nThe platform must be easy to operate by Data Analysts and stakeholders without the need of any developer.\n\nA data streaming platform typically processes a bunch of events at any given instant. After a bit of tuning and with the proper infrastructure, Apache Kafka can quickly process 2 million records. It should be \"easy\" and intuitive to display aggregated values in real-time, to provide instant insights, and be able to act upon them quickly. Also, looking at the whole history at a glance (day, week, year) is a must to see the evolution and trends.\n\n## Conduktor: an intuitive interface to peek into your data\n\nWe provide a desktop application with a friendly user interface to work with Apache Kafka and its extensions (Schema Registry, Kafka Connect, Kafka Streams, ksqlDB, etc.).\n\nConduktor enables you to look into the data published into your Apache Kafka topics. It is compatible with most data formats: from the basics to the most complex.\n\nYou can search, analyze, and export data published in real-time or from the past and analyze them. It's often a requirement for developers, QA, and Data Analysts.\n\n## Working with Conduktor\n\nLet's walk through how to work with Conduktor. We need to make sure to have the following:\n\n1. Access to a running Apache Kafka cluster.\n2. [Download](https://www.conduktor.io/download) and install Conduktor on your own machine\n\nSignup/Login into Conduktor, you will be able to synchronize your clusters with your team or create new clusters:\n\n![](//images.contentful.com/lm7q36eaz111/4emeW1vUztV1zeh4x9UXlg/7ee12485681dc3aa19f620f889b7b716/1.png)\n\nConnect Conduktor to your Apache Kafka cluster:\n\n![](//images.contentful.com/lm7q36eaz111/2yc4eYtSel0a1TqjVvooTZ/67d30e4256a33a1b7d6772fa95c913b6/2.png)\n\n### Connectivity\n\nConduktor supports on-premise installations, Kubernetes custom installations, Cloud-managed installations, as well as service providers like Confluent Cloud, Aiven, CloudKarafka, you name it.\n\nYou can validate connection details before saving them and iterate easily in case of trouble. You can connect Conduktor to other features outside of our scope here: SSH, Schema Registry, Kafka Connect, ksqlDB, etc. These are not immediately applicable to business stakeholders but are relevant for Operational needs.\n\nOnce connected, you've presented a dashboard with an overview of your Apache Kafka cluster, and important metrics to watch for. Conduktor will warn you if it detects anything wrong (misconfiguration, topics failures, Kafka Streams applications down...).\n\n![](//images.contentful.com/lm7q36eaz111/18coiYhzHpqSU0HMYiwfpi/a39eb3a2c70aad0a77a5c38e47683aec/3.png)\n\n### Data Exploration\n\nApache Kafka has its data organized in topics. It's quite normal to have hundreds of them in a small business, and thousands in larger businesses. The topics are named according to the data they serve and their level of security, privacy, business units, etc.\n\nA few examples of classic analytics data published into Apache Kafka are: user signups, notifications, and product feature clicks.\n\n![](//images.contentful.com/lm7q36eaz111/68UL9qYlPlureIUDxa5DBE/d67cd1b3cb4fc3e5dbba974a4a783cfa/4.png)\n\nYou can explore the data published on any of the topics by clicking the search icon associated with each Topic. In the above cluster, let's look into the notifications topic.\n\n### Data Lookup and Selection\n\nConduktor provides a powerful data lookup capability.\n\nIt offers flexibility to lookup data across various timeframes like current data, last hour, yesterday, since the beginning, etc. It provides support for various data formats like JSON, Apache Avro (with many variations), Protobuf, JSON-Schema, and binary data. Each topic can have a different format due to specializations, the programming language used, and developer experience.\n\nHere, a topic using the JSON format:\n\n![](//images.contentful.com/lm7q36eaz111/43aQiuLj78zUZaL3aXvBS5/138819b4142f6bc0212934190d36fde4/5.png)\n\nYou can start building insights by filtering data based on different criteria: similarity, equality, containment, specific field, and regular expressions. This is available for data and metadata of the records.\n\nHere, we're lookup for all the \"email\" notifications types only:\n\n![](//images.contentful.com/lm7q36eaz111/635KTrr9ZFOsFxyMDqKVzy/35fdacab9968566280f4d81d2d5f88d9/6.png)\n\n### Data Views\n\nConduktor provides multiple views to see the data and will keep expanding to provide more perspectives of the same data.\n\nBy default, the view is clear and simple to quickly see the data at a glance. Conduktor also provides a tabular view to display data alongside its other attributes: key, value, timestamp, and headers (deconstructed). This allows for customization (column selection) to reduce data noise by remaining focused on the useful bits of data.\n\nIt's also possible to \"project\" the data to extract only what's necessary.\n\nHere, we focus on the field notification.name by using a \"Field Selection\":\n\n![](//images.contentful.com/lm7q36eaz111/2yoXk1WLOff2enWcKLXTU5/1275e60209add65ef57178320a5c1f0e/7.png)\n\nData features are essential (aggregations in real-time, keeping the latest occurrence by key only...), and we keep adding more and more (charts, histograms, pivot tables)! Feel free to tell us what you'd like to see.\n\n### Offline Analysis\n\nOnce we have captured some data, we can export them for offline analysis and share them internally.\n\nConduktor can export data in Excel-supported CSV format or JSON for developers. It's possible to export thousands of records in real-time, or just a small slice. It will allow us to build visualization like this:\n\n![](//images.contentful.com/lm7q36eaz111/3Zeq98bbpa3YrZF5XjN17F/f04fae2efc88611270c2c4d79fd5c4fe/8.png)\n\n## Next steps\n\n[Try Conduktor for free](https://www.conduktor.io/) and let us assist you in building your Data-Driven Business.\nYou can also [ask us for a personalized demo](https://www.conduktor.io/demo) if you wish to get more insights!\n\nThank you.","author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"15jwOBqpxqSAOy2eOO4S0m","type":"Entry","createdAt":"2021-07-27T20:38:16.633Z","updatedAt":"2022-06-02T07:44:08.383Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":7,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Stéphane","surname":"Derosiaux","title":"Chief Technology Officer","twitter":"https://twitter.com/sderosiaux","github":"https://github.com/sderosiaux","linkedIn":"https://www.linkedin.com/in/st%C3%A9phane-derosiaux","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5fFFo1Rzqa8ZPG0VhWse8L","type":"Asset","createdAt":"2021-10-08T17:47:36.618Z","updatedAt":"2022-09-25T07:22:02.969Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":5,"locale":"en-US"},"fields":{"title":"Stephane Derosiaux","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5fFFo1Rzqa8ZPG0VhWse8L/ba8a3398e3326fc196029638925f94ea/stephane-d.png","details":{"size":49394,"image":{"width":192,"height":192}},"fileName":"stephane-d.png","contentType":"image/png"}}}}},"publishDate":"2021-04-07T00:00+03:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2h51Hzg7bhPHk5OXDiLXMC","type":"Entry","createdAt":"2023-03-15T08:21:14.533Z","updatedAt":"2023-03-15T08:21:14.533Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Data Integration - The What, Why, and Benefits","description":"Data Integration - The What, Why, and Benefits","noIndex":false}},"slug":"data-integration-the-what-why-and-benefits","title":"Data Integration - The What, Why, and Benefits","navbarDescription":"Data Integration - The What, Why, and Benefits","description":"In this blog we'll go over Data Integration. What is it? Why is it? What are some of the benefits? Then, let's look at the how. ","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3In906dAvfNhl57Jjo6f3g","type":"Asset","createdAt":"2023-03-15T08:20:07.143Z","updatedAt":"2023-03-15T08:20:07.143Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"SchemeRegistry OG","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3In906dAvfNhl57Jjo6f3g/41230d3d41955020499aee073cf063f8/SchemeRegistry_OG.webp","details":{"size":14678,"image":{"width":1200,"height":630}},"fileName":"SchemeRegistry_OG.webp","contentType":"image/webp"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are two ways in which data can be created: either two existing datasets are merged to create a new dataset or a whole new dataset is entered in the database. The latter approach is hard as creating new effective data is time consuming and requires a different skillset. Merging data to create new data or insights is less expensive than creating new data. What's good about merging data is that the whole process of merging data can be automated. Automation is a valuable component when it comes to scaling your business.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A data warehouse is powerful because it retrieves data from different data sources such as IoT devices and combines them to form a renewed perspective. This process of merging data from different sources is called data integration. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In the present world, big data integration is more important, as organizations are collecting and storing huge amounts of data from various sources. This data is highly valuable for business decisions, but it can also be overwhelming to deal with. Data integration helps in managing this big data, by providing a unified view of the data and making it accessible for analysis and reporting.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data integration ensures that there is a consistent flow of data from one point to another; to be able to do this it automates the ETL and ELT processes. In this article, you will learn what data integration is, its use case and different approaches used to implement it.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What Is Data Integration and Where Is It Used?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data integration is the process of combining data from multiple sources into a single, unified view. This can be accomplished in a variety of ways, depending on the specific needs of the organization and the types of data being integrated.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The data integration architecture comprises:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A network of data sources","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A master server","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Clients who access the data","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To make sure that data is up to date, data integration employs the Change Data Capture mechanism, which notices data changes in a database and then applies to the data warehouse. While data streaming integration is employed to ensure that there is a continuous flow of data integration in real time. You can get more quality data by automating the data transformation process.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data integration is important for facilitating big data. Fetching data from different sources expands the horizon for your business intelligence platforms. Having multiple data sources gives you more perspective and insights. Many companies such as Open AI need a consistent flow of fresh data from millions of data sources to keep ChatGPT running. Below is a list of use cases where data integration is used. ","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data integration plays a vital role in the field of IoT (Internet of Things) where data from various devices and sensors is collected and integrated for analysis and decision making. It is also used in various industries like healthcare, finance, and retail to gain insights from the data and make better decisions.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"One of the most common use cases for data integration is the creation of a data warehouse or business intelligence platform. In this scenario, data is extracted from operational systems, such as databases and flat files, and then transformed and loaded into a central repository. This allows organizations to gain a more comprehensive understanding of their data and to perform advanced analytics and reporting.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Another use case for data integration is customer data integration (CDI). In this scenario, data from various sources, such as customer relationship management (CRM) systems, social media, and web analytics, is integrated to create a single, unified view of the customer. This allows organizations to gain a more complete understanding of their customers and to personalize their interactions with them.","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Benefits of Integrating Data","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data integration is important for companies because it collects data from different departments to give the annual performance of the company. Automated data integration processes save the company lots of money and time. Without automated data integration companies would have to keep on manually fetching data from different sources.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Manually integrating data is costly because employees have to keep on reporting any data changes since the data is not synchronized. Also, manually integrating leads to many errors as employees may not know the exact locations of the data source and can fail to know when data is incomplete or not fresh.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are many benefits to data integration, including:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Improved data quality:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Organizations can increase the precision and completeness of their data by integrating data from many sources, which in turn enables them to make better decisions.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Efficiency gains:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" By automating many of the laborious procedures associated with combining data from several sources, companies can save time and minimize errors.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Better decision-making:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Organizations can better understand their data and base decisions on that understanding by having a unified, single view of the data.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Greater insights:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" By combining data from different sources through data integration, organizations can gain new insights and identify patterns and linkages in the data.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Better customer interactions:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Organizations can personalize their contacts with customers and deliver a better customer experience by having a single, unified view of the customer.","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"The Process of Integrating Data","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data integration utilizes many queries such as union of conjunctions and aggregation when matching data. To get a holistic view data integration uses query maps to draw the relationship between schemas and between the Data source and the target database.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"ELT and ETL are the core of data integration as they move data from the data source to the target. ETL stands for:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Extract: This involves replicating the requested data from the data source","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Load: This involves moving the replicated data from the source to the target","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Transform: This involves cleansing data to meet the desired standards. For example:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Applying mathematical functions","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Converting data types","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Modifying text strings","marks":[],"data":{}}]}]}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"ETL transforms the data before sending it over to the target. This is to make sure that the data is completely compliant with the target. ETL makes it easier to comply with security standards because you can omit any sensitive data before sending it to the target. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"On the other side, ELT extracts the data and loads into the target without transforming the data. The data will be transformed and enriched by the target. This method favors structured and unstructured data. ELT is faster than ETL because data is sent to the target without any transformations and this can happen simultaneously. ELT is highly used in business intelligence and big data analytics.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are different ways data integration ways in which data integration can be implemented and approached. Here are 3 data integration approaches you should know:","marks":[],"data":{}}]},{"nodeType":"table","data":{},"content":[{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Integrations Strategy","marks":[{"type":"bold"}],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Definition","marks":[{"type":"bold"}],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Application-based integration","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In this implementation, applications are the ones that locate, fetch, and merge data. ","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Middleware data integration","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This implementation strategy uses an intermediary component or application to make the data compatible and then merge the data.","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Manual data integration","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This approach is different from the above approaches as the user has to locate, fetch, and merge data manually from different sources. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"\n","marks":[],"data":{}}]}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Factors You Should Consider When Integrating Data","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"These factors are:","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data version:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" This determines the age of the data found in the data warehouse. You have to be very careful when choosing data that has an older version as the data is likely to be inaccurate and misleading.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data examination and matching:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" This factor determines how data will be merged. Will the data that is from the IT department data source merged with the one from the finance department data source in the computer science database? This factor is concerned about having an integration strategy or algorithm.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data specification and granularity:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" For your data integration procedure to be successful in giving groundbreaking insights and perspective you have to know how detailed and specific the data should be. Collecting what you only need is important to avoid storing any unwanted data that will drive storage costs. You need a complete idea of which data types and tables you have to retrieve for your data integrations process to produce complete insights.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Drawbacks of Using Data Integration","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data integration is very complex in a way that getting to build the infrastructure is challenging. Also new technologies are demanding. Are you going to use structured and real-time data? Your infrastructure must be flexible and ready to accept different types of data.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Integrating data from sources such as legacy systems can give you data that is incomplete.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"One of the key challenges of data integration is dealing with data quality issues. Data from different sources may be structured differently, use different codes or formats, and contain errors or inconsistencies. These issues must be identified and addressed during the integration process to ensure that the resulting data is accurate and reliable.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Another challenge of data integration is dealing with data security and privacy. Data from different sources may contain sensitive information that needs to be protected, such as personal identifying information or financial data. This requires careful planning and management to ensure that the data is properly secured and that access to it is controlled.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Overall, data integration is a complex and multifaceted process that requires careful planning, management, and execution. It is essential for organizations to have the right tools, processes, and expertise in place to effectively integrate data from multiple sources and to ensure that the resulting data is accurate, reliable, and secure.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In conclusion, data integration is a vital process that allows organizations to combine data from multiple sources into a single, unified view. This enables organizations to make more informed decisions by providing a complete, accurate, and up-to-date view of their data. Data integration is a complex and multifaceted process that requires careful planning, management.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Want to simplify your Kafka? ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Try Conduktor for free","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]}]},"publishDate":"2023-03-10T00:00+02:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1B3N0atWKcxQS9a07Pbcik","type":"Entry","createdAt":"2023-01-31T09:51:28.423Z","updatedAt":"2023-03-14T20:37:48.838Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":9,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Kafka Playground: Two Free Clusters Without the Operational Hassles","description":"We are proud to announce the launch of Conduktor Playground, giving Conduktor Cloud users access to two free Apache Kafka clusters for experimentation.","noIndex":false}},"slug":"kafka-playground-two-free-kafka-clusters-without-operational-hassles","title":"Kafka Playground: Two Free Clusters Without the Operational Hassles","navbarDescription":"We are proud to announce the launch of Playground.","description":"We are proud to announce the launch of our Playground, giving Conduktor Cloud users access to two free Apache Kafka clusters for experimentation.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2vZyBu3T0dXNo3KZ7y5EjB","type":"Asset","createdAt":"2023-02-22T19:30:11.644Z","updatedAt":"2023-02-22T19:30:11.644Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"PlaygroundAnouncement","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2vZyBu3T0dXNo3KZ7y5EjB/cd455bc47778ec076597d0770158f817/PlaygroundAnouncement.png","details":{"size":91338,"image":{"width":2048,"height":1080}},"fileName":"PlaygroundAnouncement.png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We are proud to announce the launch of our Playground, giving Conduktor Cloud users access to two free Apache Kafka clusters for experimentation. For users just starting out on their Kafka journey, the Personal Playground provides access to Conduktor and a free Kafka cluster to get familiar with both the power of Kafka and Conduktor. All without having to worry about spinning up your own Kafka clusters or using a Command line interface (CLI). For seasoned Kafka users looking to see how Conduktor can help them in their Kafka journey, the Organizational Playground provides a Kafka cluster that’s shared among teammates and enables Engineering teams to collaborate within Conduktor.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Personal Playground: Your own sandbox for learning Kafka","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If you want to learn how Kafka works today, you can either host open-source Apache Kafka or use managed Kafka services. The first option adds an administrative burden and the second has cost implications even in the learning phase. The Personal Playground is a free managed Kafka cluster where you can experiment and learn how to create Kafka topics, consume messages, produce data, and monitor consumer groups all within the Conduktor UI.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Interacting with Apache Kafka requires using the CLI. This can be a daunting process with a significant learning curve. The Conduktor Console makes working with Kafka an intuitive and interactive experience.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"vrZ6gj0R7I54pwVHI060U","type":"Asset","createdAt":"2023-02-22T14:06:35.544Z","updatedAt":"2023-02-23T16:02:06.964Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":6,"locale":"en-US"},"fields":{"title":"CreatingTopicsPlayground","description":"","file":{"url":"//videos.ctfassets.net/lm7q36eaz111/vrZ6gj0R7I54pwVHI060U/c2dc65ea960554bdd0592d183adf1b1f/PlaygroundCreateTopicsFinal.mov","details":{"size":10834691},"fileName":"PlaygroundCreateTopicsFinal.mov","contentType":"video/quicktime"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Organizational Playground: Collaborate with your teammates","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Organizational Playground gives you a Kafka cluster that’s shared among teammates. If you’re a company thinking about using Conduktor, the Organizational Playground lets you test Conduktor while collaborating with your team - all without the hassle of setting up Kafka clusters yourself. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When you’re using Kafka at scale, your worries change from how to use Kafka to how to govern Kafka. Conduktor was made to help teams experience the power of Kafka without operational and organizational pains.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5Q0DScuY36rKxCL1jXNZlL","type":"Asset","createdAt":"2023-02-22T14:07:18.600Z","updatedAt":"2023-02-22T14:07:18.600Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"RBACPlayground","description":"","file":{"url":"//videos.ctfassets.net/lm7q36eaz111/5Q0DScuY36rKxCL1jXNZlL/dfe5aed023c1d275aca25f128f13ed1f/RBACPlayground.mov","details":{"size":3467345},"fileName":"RBACPlayground.mov","contentType":"video/quicktime"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"\n","marks":[],"data":{}},{"nodeType":"text","value":"Getting Started","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To get your two free Kafka clusters, you can sign up for a 30-day Conduktor Cloud free trial ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"here","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":". When you sign up for a free trial you get access to all Conduktor features - including features only available in Conduktor Enterprise like Data Masking, and Topic-as-a-Service.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To integrate Playground with your existing applications, navigate to the Admin page and copy the connection details for your Playground connection into your application configuration.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"71aD7h6S2Pb02gx1x4I3Aq","type":"Asset","createdAt":"2023-02-22T14:08:19.176Z","updatedAt":"2023-02-22T14:08:19.176Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Admin","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/71aD7h6S2Pb02gx1x4I3Aq/bdd58c8617268d109b3b140e18201eb9/AdminPhotoBlog__1_.png","details":{"size":77459,"image":{"width":2874,"height":1466}},"fileName":"AdminPhotoBlog (1).png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conduktor","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor's Playground is a big leap forward in proliferating the use of Kafka for developers and enterprises transitioning to the cloud. It is a big step towards realizing our vision at Conduktor — Kafka should be accessible to all and any friction for developers should be minimized.","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6V3UfdQUX4TeDpfFYtFD6I","type":"Entry","createdAt":"2023-02-22T19:20:39.991Z","updatedAt":"2023-02-22T19:20:39.991Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Shashank","surname":"Golla","title":"Head of Product Marketing","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"52ohL6VuQifd4USoHGV1zX","type":"Asset","createdAt":"2023-02-22T19:20:34.399Z","updatedAt":"2023-02-22T19:20:34.399Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"1586986866218","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/52ohL6VuQifd4USoHGV1zX/c88afc8b9228f4abef654dfd09be6996/1586986866218.jpeg","details":{"size":34960,"image":{"width":800,"height":800}},"fileName":"1586986866218.jpeg","contentType":"image/jpeg"}}}}},"publishDate":"2023-02-23T00:00+03:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"74BDYeTTzGbQ1vlxAYJMEE","type":"Entry","createdAt":"2023-02-23T16:49:33.769Z","updatedAt":"2023-03-14T20:36:22.925Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Topic as a Service: Automate Governance Decisions for Apache Kafka","description":"topic-as-a-service-automate-governance-decisions-for-apache-kafka","noIndex":false}},"slug":"topic-as-a-service-automate-governance-decisions-for-apache-kafka","title":"Topic as a Service: Automate Governance Decisions for Apache Kafka","navbarDescription":"Topic as a Service was built to solve governance issues for Apache Kafka. Find out how in this blog","description":"Kafka's widespread usage has introduced problems related to governance, as different teams combine their Kafkas. Topic as a Service for the Conduktor Platform was built to help solve these issues.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5hWT5hYoVq0CdMY3j53Osh","type":"Asset","createdAt":"2023-01-31T09:51:25.080Z","updatedAt":"2023-02-22T19:29:09.645Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"TAASDeepdive OG","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5hWT5hYoVq0CdMY3j53Osh/a206e239f536068c3760784e9bcbfba1/TAASDeepdive_OG.webp","details":{"size":8814,"image":{"width":1200,"height":630}},"fileName":"TAASDeepdive_OG.webp","contentType":"image/webp"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There is a good reason for Apache Kafka's rapid spread around the world's major organizations. It is a great system, flexible and powerful enough to handle various real-time data needs. However, the wide usage of Kafka in organizations will also introduce problems related to security and governance. Topic as a Service for the ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Conduktor Platform","marks":[],"data":{}}]},{"nodeType":"text","value":" was built to help solve these issues.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"fb80YPetIqd6iRksdIMJM","type":"Asset","createdAt":"2023-01-31T09:52:57.700Z","updatedAt":"2023-01-31T09:52:57.700Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Topic as a Service","description":"Dashboard for topic as a service","file":{"url":"//images.ctfassets.net/lm7q36eaz111/fb80YPetIqd6iRksdIMJM/c8761d2db8b1576ba57dd7283e7ee957/screenshot_2023-01-13_at_11-36-34_topic_governance_1_.png","details":{"size":86788,"image":{"width":1920,"height":1080}},"fileName":"screenshot_2023-01-13_at_11-36-34_topic_governance_1_.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Why Do You Need Topic as a Service?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When starting something new, most don't take the time to check everything. Few people read terms & conditions or do test runs. People want to get started. So it is with Apache Kafka; deciding on naming conventions or figuring out the best topic configuration will slow you down, and you can always look at them later. Not a problem when you're getting started, but as Kafka scales, it will become important.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"At some point, organizations will want to combine their different Kafka flavors and teams. All of the different approaches will not work together without a serious amount of effort to integrate them. Soon enough, you are spending more time building and maintaining these integrations than you are actually working with Kafka!","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There is one potential solution to this: just get Ops to handle the foundational topic processes. However, given how much of Kafka is built on topics, your Ops will change from occasional gatekeepers to permanent topic robots, doing nothing but approving requests. It's fair to say that this is not the best use of your skilled employees.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What Topic as a Service Can Bring","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The appeal of Topic as a Service is thus quite simple: it becomes your Kafka gatekeeper, standardizing topic access and creation right from the start. It's an effortless approach to ensuring that disparate teams can create, share and control Kafka resources.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let's take a look at how Topic as a Service can simplify topic ownership and access. In the clip below, the \"Wikipedia\" application takes ownership of any topic prefixed with \"wikipedia\", auto-generating ACLs for each topic:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3VssFeBIhxs6oaqBjXD82U","type":"Asset","createdAt":"2023-01-31T09:53:45.060Z","updatedAt":"2023-01-31T09:53:45.060Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"TaaS","description":"https://images.ctfassets.net/lm7q36eaz111/3Qp4jwtPuuMgl56rglPm8a/c9d27cb7dcb88767bd37d36bb39e92fb/VideoThumbnail_TaaS__1_.png","file":{"url":"//videos.ctfassets.net/lm7q36eaz111/3VssFeBIhxs6oaqBjXD82U/566da525fc61f3578afc239b00daa26f/TaaS.mp4","details":{"size":997247},"fileName":"TaaS.mp4","contentType":"video/mp4"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If any other applications need access to these topics, all that is needed is to send a request and have it granted:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2dpX9z2EHXGG6c7AvyK2Vs","type":"Asset","createdAt":"2023-01-31T09:54:39.043Z","updatedAt":"2023-01-31T09:54:39.043Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"bi-taas","description":"https://images.ctfassets.net/lm7q36eaz111/3Qp4jwtPuuMgl56rglPm8a/c9d27cb7dcb88767bd37d36bb39e92fb/VideoThumbnail_TaaS__1_.png","file":{"url":"//videos.ctfassets.net/lm7q36eaz111/2dpX9z2EHXGG6c7AvyK2Vs/86b9f77e72ed1dde94ab85503971b9a9/bi-taas.mp4","details":{"size":2094516},"fileName":"bi-taas.mp4","contentType":"video/mp4"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Topic as a Service is still in development, so not every feature is ready to be demonstrated, but the following features will soon be available:","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Upcoming Features","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Automated Topic Creation","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Applications and Teams created in Topic as a Service will be able to define naming schemes and topic configurations, allowing authorized users to instantly create their own topics with full approval of application owners","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Topic Creation bound to Identity","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With Topic as a Service, all topic creation is connected back to the individual user's identity, allowing you to understand the flow of requests, who created what, who asked for what, and how it is all being shared","marks":[],"data":{}}]}]}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"How do I get Started","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Topic as a Service is available to Enterprise subscribers for the Conduktor Platform. Before you start creating applications, there are two steps you'll need to take:","marks":[],"data":{}}]},{"nodeType":"ordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Create your environments (Prod, Staging, etc.) and assign them to clusters. This is done from within Topic as a Service.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Create the teams that will own each application. Teams can be created during application setup, but you will need to go to the Admin panel to assign users to each team.","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For those without subscriptions, you can try out Topic as a Service in ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://demo.conduktor.io/home/"},"content":[{"nodeType":"text","value":"our demo environment","marks":[],"data":{}}]},{"nodeType":"text","value":" - environments and teams already exist, so you can immediately try out the features.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Summary","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Topic as a Service is one of the simpler solutions available in the Conduktor Platform, but it is no less important for that. The very best advances in productivity have never been highly complicated; instead, they take what used to be tedious human work and automate it, freeing up time to focus on other, more important things. So it is with Topic as a Service, giving time back to your teams and letting them work on what matters, instead of worrying about procedures and rules.","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"15jwOBqpxqSAOy2eOO4S0m","type":"Entry","createdAt":"2021-07-27T20:38:16.633Z","updatedAt":"2022-06-02T07:44:08.383Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":7,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Stéphane","surname":"Derosiaux","title":"Chief Technology Officer","twitter":"https://twitter.com/sderosiaux","github":"https://github.com/sderosiaux","linkedIn":"https://www.linkedin.com/in/st%C3%A9phane-derosiaux","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5fFFo1Rzqa8ZPG0VhWse8L","type":"Asset","createdAt":"2021-10-08T17:47:36.618Z","updatedAt":"2022-09-25T07:22:02.969Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":5,"locale":"en-US"},"fields":{"title":"Stephane Derosiaux","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5fFFo1Rzqa8ZPG0VhWse8L/ba8a3398e3326fc196029638925f94ea/stephane-d.png","details":{"size":49394,"image":{"width":192,"height":192}},"fileName":"stephane-d.png","contentType":"image/png"}}}}},"publishDate":"2023-02-01T00:00+02:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3xRRFhI5fXTQ67Mwu7eXU","type":"Entry","createdAt":"2023-01-10T11:03:56.211Z","updatedAt":"2023-01-10T14:29:32.617Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"KRaft Quorum: How to run Kafka without Zookeper","description":"In this article, we’ll discuss what Zookeeper is, what its role is, its limitations, and how KRaft mode mitigates those limitations.","noIndex":false}},"slug":"kraft-quorum-run-kafka-without-zookeeper","title":"KRaft Quorum: How to run Kafka without Zookeeper","navbarDescription":"In this article we’ll discuss what Zookeeper is, what its role is, its limitations, and how KRaft mode mitigates those limitations.","description":"In this article, we’ll discuss what Zookeeper is, what its role is, its limitations, and how KRaft mode mitigates those limitations.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1b5wTdUgEgdZNEVPNDTYLu","type":"Asset","createdAt":"2023-01-10T10:50:51.329Z","updatedAt":"2023-01-16T13:25:13.167Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Zookeeper OG","description":"Kafka KRaft mode","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1b5wTdUgEgdZNEVPNDTYLu/8cd537243cfb282ad073dc01a89963ca/Zookeeper_OG__1_.png","details":{"size":58753,"image":{"width":1200,"height":630}},"fileName":"Zookeeper_OG (1).png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"From the release of Apache Kafka 2.8.0, users have had early access to ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://cwiki.apache.org/confluence/display/KAFKA/KIP-500%3A+Replace+ZooKeeper+with+a+Self-Managed+Metadata+Quorum"},"content":[{"nodeType":"text","value":"KIP-500","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", which removes the Zookeeper dependency from Kafka. The release of Apache Kafka 3.3.0 saw KIP-500 become production-ready, meaning Kafka now relies on an internal Raft quorum that uses a Kafka Topic to store metadata, and some Kafka servers act as Controllers. The new feature simplifies cluster administration and infrastructure management and makes the Kafka cluster more scalable.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In this article, we’ll first discuss what Zookeeper is. Next, we'll see what its role is in Kafka. Finally, what are the limitations of Zookeeper, and how does KRaft mode mitigate those limitations?","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Zookeeper in Kafka","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"What is Zookeeper","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Zookeeper is a service for maintaining configurations, naming, and distributed synchronization. Zookeeper allows distributed processes to coordinate with each other through zNodes. The ","marks":[],"data":{}},{"nodeType":"text","value":"performance aspects ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"of Zookeeper allow it to be used in large distributed systems. The ","marks":[],"data":{}},{"nodeType":"text","value":"reliability aspects ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"prevent it from becoming the single point of failure in big systems. Its ","marks":[],"data":{}},{"nodeType":"text","value":"strict ordering ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"allows sophisticated synchronization primitives to be implemented at the client.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The main difference between Zookeeper and standard file systems is that every zNode can have data associated with it. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Zookeeper was designed to store coordination data: status information, configuration, location information, etc. This kind of meta-information is usually measured in kilobytes, if not bytes. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Zookeeper provides multiple features for distributed systems:","marks":[],"data":{}}]},{"nodeType":"ordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Distributed configuration management","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Self-election / Consensus building","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Coordination and locks","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Key value store","marks":[],"data":{}}]}]}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3fBDJD7hlvmQodn7f3AO5P","type":"Asset","createdAt":"2023-01-10T10:03:13.372Z","updatedAt":"2023-01-18T17:14:54.285Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":5,"locale":"en-US"},"fields":{"title":"WhatIsZookeeper","description":"Zookeeper Diagram","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3fBDJD7hlvmQodn7f3AO5P/6a720236474881524d0b60bce70397f4/WhatIsZookeeper.png","details":{"size":31772,"image":{"width":1920,"height":590}},"fileName":"WhatIsZookeeper.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The service itself is replicated over a set of machines that comprise it. These machines maintain an in-memory image of the data tree along with transaction logs and snapshots in a persistent store. Because the data is kept in memory, Zookeeper is able to get very high throughput and low latency numbers. The downside to an in-memory database is that the size of the database that Zookeeper can manage is limited by memory.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As long as a majority of the servers are available, the Zookeeper service will be available.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Zookeeper Quorum","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Zookeeper needs to have a strict majority of servers to form a consensus when votes happen. Therefore a Zookeeper quorum can have 1, 3, 5, 7, and up to (2N+1) servers. This allows 0, 1, 2, 3, and N servers to go down without making the cluster unusable.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2F1Zdp6jXhTV2PK9FSjE2k","type":"Asset","createdAt":"2023-01-10T10:05:53.885Z","updatedAt":"2023-01-18T17:12:10.861Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"zookeeperquorum","description":"Zookeeper Quorum explanation","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2F1Zdp6jXhTV2PK9FSjE2k/7ae0f500a4ac695fe3694fb2eced4cb3/Frame_21.png","details":{"size":96623,"image":{"width":1919,"height":1147}},"fileName":"Frame 21.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Role of Zookeeper in Kafka","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Here are the things Zookeeper does for Kafka; actually, for any distributed system-","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Brokers registration, with a heartbeat mechanism to keep the broker list updated","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Maintaining metadata of topics","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Topic configuration (partitions, replication factor, additional configs, etc.)","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The list of in-sync replicas for partitions","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Performs leader election for all partitions","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Store metadata of Access Control Lists (ACLs), if security is enabled","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Topics","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Consumer Groups","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Users","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Store the Kafka cluster-id (randomly generated while the Broker registers the first time)","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Zookeeper: Limitations and Challenges","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Zookeeper seems an inseparable part of running a distributed system like Kafka but it has some limitations which are preventing Kafka from working to its full potential. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let’s discuss some of those challenges here-","marks":[],"data":{}}]},{"nodeType":"ordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Two different systems","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" - Zookeeper is an external system to Kafka which means that it comes with its own configuration syntax, management tools, etc. Therefore, if you want to deploy a Kafka cluster then you would also have to manage, deploy, and monitor Zookeeper as well.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Error Prone","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"- These two distributed systems require different configurations and the level of complexity increases, thus it is much easier for system administrators to make mistakes.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Duplication of work","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"- Having two systems also leads to duplication of work— for example, in order to enable security features you would have to apply the relevant configuration to both services.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Additional Processes","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"- Having an external metadata store is inefficient regarding resources as you need to run additional processes.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Limits Kafka Scalability-","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" With Zookeeper, Kafka scalability is limited. Every time the cluster is starting up, the Kafka controller must load the state of the cluster from Zookeeper.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Fewer Partitions in the Cluster","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"- The same happens when a new controller is elected. Given that the amount of metadata gets bigger over time, this means that the loading of such metadata becomes more inefficient over time and thus limiting the number of partitions the cluster can store.","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Zookeeper-based Kafka controller loads all the metadata synchronously on startup and during this time the cluster is unavailable. Also, it has to send all metadata to all the brokers.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"KRaft - Kafka Raft","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To overcome the challenges of Zookeeper in Kafka, a KIP (Kafka Improvement Plan) was submitted for Zookeeper-less Kafka (","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://cwiki.apache.org/confluence/display/KAFKA/KIP-500%3A+Replace+ZooKeeper+with+a+Self-Managed+Metadata+Quorum"},"content":[{"nodeType":"text","value":"KIP-500","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":") and ","marks":[],"data":{}},{"nodeType":"text","value":"version 2.8.0 ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"introduced early access to Zookeeper-less Kafka, with KRaft becoming production-ready in ","marks":[],"data":{}},{"nodeType":"text","value":"version 3.3.0","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". In the latest release, ZooKeeper can be replaced by an internal Raft quorum of controllers.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"How does it Work?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When Kafka Raft Metadata mode is enabled, Kafka will store its metadata and configurations into an internal topic called ","marks":[],"data":{}},{"nodeType":"text","value":"__cluster_metadata","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". This topic has a single partition and the leader will be on the active controller. This internal topic is managed by the internal quorum and replicated across the cluster. The nodes of the cluster can now serve as brokers, controllers, or both (called combined nodes).","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When ","marks":[],"data":{}},{"nodeType":"text","value":"KRaft mode ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"is enabled, only a few selected servers can serve as controllers and these will compose the internal quorum. The controllers can either be in active or standby mode which will eventually take over if the currently active controller server fails or is taken down.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6hGmhRLWAyvAzfaU34f3qN","type":"Asset","createdAt":"2023-01-10T10:10:27.469Z","updatedAt":"2023-01-18T17:12:32.479Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"kafkaRaft","description":"Kafka Raft diagram","file":{"url":"//images.ctfassets.net/lm7q36eaz111/6hGmhRLWAyvAzfaU34f3qN/9e4a469ae8411940d050c53439ce0d13/Frame_236.png","details":{"size":40824,"image":{"width":1920,"height":552}},"fileName":"Frame 236.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As we can see in the above diagram, Zookeeper nodes are replaced by KRaft Controllers. The starting up of the controllers will have the following process-","marks":[],"data":{}}]},{"nodeType":"ordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Some Kafka Brokers will be designated as KRaft controllers","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"These controllers will elect a leader","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The leader will have all previously committed records","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka brokers will fetch metadata from this KRaft leader.","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Now every Kafka server comes with an additional configuration parameter called ","marks":[],"data":{}},{"nodeType":"text","value":"process.roles","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". This parameter can take the following values:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"broker","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": The Kafka server shall serve as a broker","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"controller","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": The Kafka server shall serve as a controller of the internal Raft quorum","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"broker,controller","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": The Kafka server shall serve as both a controller of the quorum and a broker","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When ","marks":[],"data":{}},{"nodeType":"text","value":"process.roles ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"is not configured, it is assumed that the cluster will run in Zookeeper mode. ","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"How does it overcome the challenges with Zookeeper?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Now let’s discuss how KRaft mode can overcome some of the challenges with Zookeeper.","marks":[],"data":{}}]},{"nodeType":"table","data":{},"content":[{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Scenario","marks":[{"type":"bold"}],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With Zookeeper","marks":[{"type":"bold"}],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Problem","marks":[{"type":"bold"}],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Solution: with KRaft","marks":[{"type":"bold"}],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Controller Failover ","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"After a controller wins the election, it fetches all the topics and partitions and sends ","marks":[],"data":{}},{"nodeType":"text","value":"LeaderAndISR ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"and ","marks":[],"data":{}},{"nodeType":"text","value":"UpdateMetaData ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"to all the brokers","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let’s consider you have a huge cluster with lots of partitions: this process can take a lot of time and the cluster will be unavailable during this time","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With KRaft mode, once a controller wins the election it starts getting the request from the brokers so there’s no wait time on loading all the topics, etc. again","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Node Restart","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A rolling node starts with no metadata and can only be available once it fetches all the metadata","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Fetching metadata can take longer when there are many partitions available in the cluster, which will make the node become unavailable for some time","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With KRaft mode, the rolling node needs to fetch only the metadata that it doesn’t have in its local cache. It does so using the metadata offset","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Zookeeper Dependency","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka and ZooKeeper are two distinct services, so in order to administer a Kafka cluster one needs to understand how Zookeeper works as well","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Understanding of additional service i.e. Zookeeper. One has to set up security, monitoring, etc. for Zookeeper as well in order to secure the cluster","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With KRaft mode, Kafka has been unified and it is not dependent on Zookeeper to store the metadata. The learning curve will be shortened and engineers can now focus only on Kafka","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Scalable Architecture","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In ZooKeeper mode, Kafka had to store its metadata in zNodes. Every time the cluster was starting up or a controller election was happening, Kafka Controllers had to read all the metadata from the Zookeeper service which was inefficient","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Cluster unavailability during the fetching of metadata resulting in hosting limited partitions per cluster","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With KRaft mode, deployments can now support more partitions","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Easy Setup","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"One has to configure Zookeeper as well in order to run Kafka. This is required even for doing a small proof of concept","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Running extra services even for a small setup","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With KRaft mode, you no longer have to start multiple processes","marks":[],"data":{}}]}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Migration Strategy: Zookeeper to KRaft","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Starting with KRaft is easy and one can setup the Kafka cluster in no time, but what about when the existing cluster runs with Zookeeper? How will the migration happen? What would be the operational changes required?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Answers to all these questions are available in ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://cwiki.apache.org/confluence/display/KAFKA/KIP-866+ZooKeeper+to+KRaft+Migration"},"content":[{"nodeType":"text","value":"KIP-866","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":". This Kafka Improvement Plan lists the approach to migrate from Zookeeper quorum to Raft quorum without any impact to partition availability and with minimal impacts on operators and client applications. The implementation will be available as early access starting from Apache Kafka version 3.4.0 (Check the ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://cwiki.apache.org/confluence/display/KAFKA/Release+Plan+3.4.0"},"content":[{"nodeType":"text","value":"Release plan","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":").","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Migration itself is a vast topic and still under development but the following are the steps to migrate from Zookeeper to Raft quorum-","marks":[],"data":{}}]},{"nodeType":"ordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Preparing the cluster","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" - Upgrade the cluster to the version where migration-related implementation is available. This release will have additional configurations required by Zookeeper and Kraft Controllers in order to communicate with each other during the migration phase.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Controller migration","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" - The migration only supports dedicated KRaft controllers as the target deployment. A new set of nodes will be provisioned to host the controller quorum.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Broker Migration","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" - The brokers are restarted one by one in KRaft mode. During this rolling restart, if anything is not correct there will be an option to roll back the brokers in Zookeeper mode.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Finalizing the Migration","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" - Once the cluster has been fully upgraded to KRaft mode, the controller will still be running in migration mode and making dual writes to KRaft and ZK. Once the operator has decided to commit to KRaft mode, the final step is to restart the controller quorum and take it out of migration mode by setting ","marks":[],"data":{}},{"nodeType":"text","value":"zookeeper.metadata.migration.enable","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" to \"false\" (or unsetting it)","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The removal of Zookeeper dependency is a huge step forward for Kafka. The new KRaft mode feature will extend the scalability capabilities of Apache Kafka and also shorten the learning curve since now teams won’t have to worry about ZooKeeper any longer. Now, system administrators will find it ","marks":[],"data":{}},{"nodeType":"text","value":"easier to monitor, administer, and support Kafk","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"a. Developers will have to deal with a single security model for the whole system. Moreover, we have a lightweight single-process deployment to get started with Kafka.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The roadmap for Kafka Raft and Zookeeper removal-","marks":[],"data":{}}]},{"nodeType":"table","data":{},"content":[{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka Version","marks":[{"type":"bold"}],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Release Summary(for KRaft and Zookeeper)","marks":[{"type":"bold"}],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"2.8.0","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"First KRaft Release (For development and POC purposes)","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"3.3.0","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"KRaft is now Production-ready with this version","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"3.4.0*","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Upgrade from Zookeeper (early access)","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"3.5.0*","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Zookeeper mode will be deprecated","marks":[],"data":{}}]}]}]},{"nodeType":"table-row","data":{},"content":[{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"4.0.0*","marks":[],"data":{}}]}]},{"nodeType":"table-cell","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Removal of Zookeeper mode","marks":[],"data":{}}]}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"* Denotes future versions","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For users of Conduktor Platform, all versions of Apache Kafka continue to be supported, meaning you can continue to use Zookeeper quorum, or switch to Raft quorum without any issues. If you haven't yet tried Conduktor Platform, ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"you can sign up for free and try it out now","marks":[],"data":{}}]},{"nodeType":"text","value":". If you don't want to put up with the hassle of Zookeeper, why accept any Kafka flaws when you don't have to?","marks":[],"data":{}}]}]},"publishDate":"2023-01-10T14:00+00:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3SnTX5oAyQ9wSkBmvQOZHU","type":"Entry","createdAt":"2023-01-04T15:38:56.120Z","updatedAt":"2023-01-04T16:40:34.795Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Conduktor Monitoring and Alerting - Deep Dive","description":"Take a look at the solutions for Kafka monitoring and alerting in the Conduktor Platform and see what sets them apart.","noIndex":false}},"slug":"conduktor-monitoring-and-alerting-deep-dive","title":"Conduktor Monitoring and Alerting - Deep Dive","navbarDescription":"Take a look at the solutions for Kafka monitoring and alerting in the Conduktor Platform and see what sets them apart.","description":"Monitoring and Alerts are vital for the long term health of systems, and Kafka is no different. Unfortunately, no solutions exist that have been configured especially for Kafka - until now. Conduktor Platform delivers seamless monitoring and instant alerts.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7x87Is1OzdIxw500e2ccUU","type":"Asset","createdAt":"2023-01-04T16:40:24.909Z","updatedAt":"2023-01-04T16:40:24.909Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"MonitoringDeepdive OG","description":"Monitoring Deep Dive","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7x87Is1OzdIxw500e2ccUU/6881ea2eea8e96270999bdbabd1fbce1/MonitoringDeepdive_OG.png","details":{"size":252095,"image":{"width":1200,"height":630}},"fileName":"MonitoringDeepdive_OG.png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We've covered the ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/conduktor-console-the-evolution-of-kafka-user-interfaces/"},"content":[{"nodeType":"text","value":"Conduktor Console","marks":[],"data":{}}]},{"nodeType":"text","value":", which gives you a powerful user interface for Apache Kafka, and ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/conduktor-testing-deep-dive/"},"content":[{"nodeType":"text","value":"Testing","marks":[],"data":{}}]},{"nodeType":"text","value":", allowing you to test anything you build. But once you've got Kafka applications into production, how do you ensure that everything remains stable and functional? Developers rely on monitoring and alerting software to keep them up to date and aware of problems with applications. There are plenty of very good monitoring solutions already available and widely used, but none have been designed for Apache Kafka. That's why we built ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/monitoring/"},"content":[{"nodeType":"text","value":"Conduktor Monitoring","marks":[],"data":{}}]},{"nodeType":"text","value":" and ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/alerting/"},"content":[{"nodeType":"text","value":"Alerting","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"qnQVFiBH72BRXmEI7yXeV","type":"Asset","createdAt":"2023-01-04T15:39:50.172Z","updatedAt":"2023-01-09T11:47:05.036Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Monitoring Overview","description":"Conduktor Monitoring overview screen","file":{"url":"//images.ctfassets.net/lm7q36eaz111/qnQVFiBH72BRXmEI7yXeV/688836995c0e13d9294e4cd7c9774c03/image02__1_.png","details":{"size":139158,"image":{"width":2049,"height":1354}},"fileName":"image02 (1).png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Pitfalls of Existing Monitoring Solutions","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are some monitoring solutions that work with Kafka. Many development teams will make use of open-source tools like Grafana or Prometheus, and there are paid solutions. All have their own strengths and weaknesses, but the crucial issue is that they were not designed with Kafka in mind.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are more than 20,000 metrics you could keep track of in Kafka. Picking out the ones that matter requires real expertise and none of the general-purpose monitoring options can offer any help.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Monitoring needs to take place both over your Kafka system, keeping track of things like cluster health & broker status; and it is needed for Kafka applications, observing how an application behaves, whether it is consuming or producing messages, what its throughput is, and so on.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Assuming that you can solve these problems, building this monitoring to cover everything will take a huge amount of resources, whether time or money. It's a potentially major investment with no guarantee that the final product will be monitoring the metrics you want and need.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor Monitoring solves these problems.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What Conduktor Monitoring & Alerting can bring","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor is filled with Kafka experts who know the ins and outs of the system. There are few companies better placed to work out what Kafka metrics are important, and we have designed a system that focuses on what really matters. The metrics that are highlighted in our monitoring dashboards and in alerts are the ones that make a difference. We are an opinionated Monitoring provider.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"41v6wQE1JhwjPQSDdAsaaG","type":"Asset","createdAt":"2023-01-04T15:40:48.854Z","updatedAt":"2023-01-09T11:47:24.130Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Monitoring - Graphs","description":"Conduktor Monitoring shows issues","file":{"url":"//images.ctfassets.net/lm7q36eaz111/41v6wQE1JhwjPQSDdAsaaG/fae7072d0f66d0777d74b14c9633e5a4/image03__1_.png","details":{"size":142659,"image":{"width":2049,"height":1354}},"fileName":"image03 (1).png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Dashboarding requires no setup. As soon as you add a topic and start producing or consuming data, Conduktor Monitoring can be active, tracking metrics. Integrations with other centralized monitoring solutions are available out-of-the-box, enabling users to instantly connect Kafka monitoring to the rest of their dashboards, instead of spending weeks building their own.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are 3 main areas that are needed for monitoring and which are key to Conduktor Monitoring - cluster health, topic monitoring, and app monitoring. Existing Kafka monitoring focuses on Kafka itself, not really on the applications connected to Kafka. despite their importance.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Cluster health looks at metrics like active brokers, active partitions, and active controller accounts. It also collects cap usage as well as top connections over time.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3a0tUTWgOfnpcfAC2Gm72J","type":"Asset","createdAt":"2023-01-04T15:45:35.032Z","updatedAt":"2023-01-06T13:56:07.822Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Cluster Health - Activity","description":"Cluster health in Conduktor monitoring","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3a0tUTWgOfnpcfAC2Gm72J/9a8599a5a0022370e07cace21a68e53d/image05.png","details":{"size":332885,"image":{"width":2049,"height":1209}},"fileName":"image05.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Topic monitoring looks at the obvious: messages in, messages out; bits and bytes out; how much data you're storing on your topic. And we can break this down by topic or partition. So if you have a really big partition and you want to diagnose why it's always lagging, you can. It also shows you duplicates and distinct duplicate accounts. ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7BhudgYYUApti5IAu6ptYP","type":"Asset","createdAt":"2023-01-04T15:50:11.515Z","updatedAt":"2023-01-06T13:56:26.416Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Topic Monitoring","description":"Topic monitoring in Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7BhudgYYUApti5IAu6ptYP/30041d5abb3e59337b18202f065d99e7/image04.png","details":{"size":352537,"image":{"width":2049,"height":1209}},"fileName":"image04.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The App monitoring is focused around lag: lag in seconds, lag in messages, and so on. What applications actually care about are messages being processed, and we graph that over time. Everything in here is graphed over time.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"16SI12rGbtLp15cLQSo0yn","type":"Asset","createdAt":"2023-01-04T15:51:05.421Z","updatedAt":"2023-01-06T13:56:50.856Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":" App Monitoring","description":"App monitoring in Conduktor platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/16SI12rGbtLp15cLQSo0yn/5bc2c0491e70bffbd87a08dc4585544c/image07.png","details":{"size":349359,"image":{"width":2049,"height":1209}},"fileName":"image07.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Alerts are self-service and configurable at the metric level, enabling users to have fine-grained control over what comes to their attention and what can be left alone. When alerts are needed, Conduktor delivers them in an easy-to-understand form, getting to the heart of the issue. The intention of Conduktor Alerting is that any Kafka user can take action from an alert, rather than funneling everything to Ops or Senior devs. ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4GNIqEhzQZG4cEfTnJRHbi","type":"Asset","createdAt":"2023-01-04T15:52:03.170Z","updatedAt":"2023-01-06T13:57:14.456Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Alerts","description":"Creating an alert","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4GNIqEhzQZG4cEfTnJRHbi/379dd6930020310927f4343a10bcabd0/image06.png","details":{"size":340769,"image":{"width":2049,"height":1354}},"fileName":"image06.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2VtkaXWVHs9kvIX2xH44Cy","type":"Asset","createdAt":"2023-01-04T16:41:08.542Z","updatedAt":"2023-01-06T13:57:31.803Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Slack Mockup","description":"Slack Alerts","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2VtkaXWVHs9kvIX2xH44Cy/4bb77f2f302ed6ba2acdcb0f6e9e0e26/image01.png","details":{"size":316428,"image":{"width":2049,"height":1352}},"fileName":"image01.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Monitoring and Alerting just work, fitting straight into the Conduktor Platform. At any time you need you can add Monitoring and Alerting to your existing Kafka UI with zero disruption. The tests, monitoring, security features, administrator features, data quality, and everything else that belongs to the Conduktor Platform all fit together with no fuss.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor's monitoring and alerting will integrate with existing systems to enable you to centralize and avoid adding yet another tool for your Ops to look at. Currently, the following out-of-the-box integrations are available for alerts, with additional integrations in development:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Alerting Integrations","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Slack","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Webhooks (coming soon)","marks":[],"data":{}}]}]}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"How do I get Started","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Currently, Monitoring and Alerting are only available for the Docker version of Conduktor, but they are coming very soon to Conduktor Cloud. If you want to get a quick overview of the dashboards and metrics available, Monitoring is available in the ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://demo.conduktor.io/home/"},"content":[{"nodeType":"text","value":"Conduktor demo environment.","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Setting up Monitoring requires the download and configuration of the Monitoring agent, which can be done in just a few lines. After that, everything is ready. Monitoring will automatically start on your clusters, topics, and applications. Alerts are similarly simple, with a number of built-in alerts that can be configured for Slack in under a minute.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Summary","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For a deep dive blog, there hasn't been a huge amount of diving into technical details here. There's a good reason for that - with Conduktor Monitoring and Alerting, you won't need to know all the technical details. Implementation is practically instant. We surface the metrics that matter to you and your applications. Integrations are plug-and-play. Alerts are delivered in real-time, and like the dashboards only highlight what is important. What else needs to be said?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Start using Conduktor for free today and see how easy Monitoring can be.","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1JmKff2d8J5C93f4yUSfCV","type":"Entry","createdAt":"2021-12-08T13:19:31.889Z","updatedAt":"2021-12-08T13:19:31.889Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"James","title":"Technical Product Manager","linkedIn":"https://www.linkedin.com/in/james-white-301b5550/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"65GcHlPUFtafUImTScSJjv","type":"Asset","createdAt":"2021-12-08T13:18:31.985Z","updatedAt":"2022-09-25T14:00:19.941Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"james","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/65GcHlPUFtafUImTScSJjv/d3f2ce92a6a6e7c8b4abc37b3da6fafa/james.png","details":{"size":47354,"image":{"width":192,"height":192}},"fileName":"james.png","contentType":"image/png"}}}}},"publishDate":"2023-01-05T02:00+00:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2qJpgLgPZq7gNN9Cl19gZu","type":"Entry","createdAt":"2022-12-08T15:31:01.567Z","updatedAt":"2022-12-21T10:52:58.427Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Data Lakes vs Data Warehouses","description":"Explore the difference between the two storage paradigms of data lakes and data warehouses. Learn the pros and cons and where to use each one","noIndex":false}},"slug":"data-lakes-vs-data-warehouses","title":"Data Lakes vs. Data Warehouses","navbarDescription":"Explore the two dominant storage paradigms, their pros and cons, and where each is best suited.","description":"Explore the difference between the two storage paradigms of data lakes and data warehouses. Learn the pros and cons of each and which situations they are suited for.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5H4Ydev5WxL32njqPKBEa2","type":"Asset","createdAt":"2022-12-21T10:52:48.151Z","updatedAt":"2022-12-21T10:52:48.151Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"DataWarehouseVsDataLake OG","description":"Data Warehouses vs Data Lakes","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5H4Ydev5WxL32njqPKBEa2/48c9a88b958a955b999f4331b5c761bc/DataWarehouseVsDataLake_OG.png","details":{"size":67089,"image":{"width":1200,"height":630}},"fileName":"DataWarehouseVsDataLake_OG.png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are many ways to store data, and each method has its own pros and cons. You need a strategy that balances cost, accessibility, and availability. Data warehouses and data lakes are widely used as storage platforms. Yes, they are rivals, but that does not mean that they cannot be used together. Many factors and features have to be considered when comparing the two storage platforms.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In this article, you will learn the difference between a data warehouse and a data lake by exploring their functions, benefits, and drawbacks. In addition, you will learn how to choose between a data lake and a data warehouse by looking at factors such as cost and performance.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is a Data Lake?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A data lake is basically a storage repository that stores all raw unstructured, or semi-structured data from different sources in the same place. There are many benefits to creating a data lake instead of keeping your data in separate siloed locations.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"iwyMuLSvYQnTPe9XSuLB2","type":"Asset","createdAt":"2022-12-21T10:18:04.876Z","updatedAt":"2023-01-18T17:19:36.410Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Data lake diagram","description":"Data Lakes","file":{"url":"//images.ctfassets.net/lm7q36eaz111/iwyMuLSvYQnTPe9XSuLB2/e9af6ff079649496a9240b3ebeb04197/Frame_229.png","details":{"size":30003,"image":{"width":1920,"height":384}},"fileName":"Frame 229.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The above diagram shows data flowing from the two sources, Twitter and an IoT sensor into the data lake. The data from the data lake is used for machine learning and data discovery.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data lakes can be used to speed up data analytics and simplify data management. Data lakes add scalability to your data facilities because data can be received at different speeds, types, and from different locations whether on-premise or cloud. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data lakes are widely used in finance to store data that is used to build machine learning models. The media industry uses data lakes to create video recommendations for users. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data stored in the data lake can be used to improve research development and use customer data through CRMs.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"SQL queries and big data analytics are used to fetch useful information from data lakes. The data can be processed using Python and third-party data analytics applications. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You can save data as operational databases that will allow you to secure your data through encryption and other methods. Data lakes use object storage and flat architecture to store data. Metadata tags and unique identifiers are used to locate data. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The flexibility of data lakes allows you to use the analytics tool and framework of your choice such as ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://prestodb.io/"},"content":[{"nodeType":"text","value":"Presto","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" or ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.techtarget.com/searchdatamanagement/definition/Hadoop-data-lake#:~:text=A%20Hadoop%20data%20lake%20is,images%20and%20social%20media%20posts."},"content":[{"nodeType":"text","value":"Apache Hadoop","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":". ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"More importantly, a data lake makes it possible to combine unstructured and semi-structured data stored in different sizes together. This is especially important because more than ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.mongodb.com/unstructured-data#:~:text=From%2080%25%20to%2090%25%20of,used%20to%20guide%20business%20decisions."},"content":[{"nodeType":"text","value":"80% of the world’s data is unstructured","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":". A data lake can help you sift through this unstructured data and make sense of it in order to gain a deeper understanding of your users, product, and business. ","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"What Are the Benefits of Using a Data Lake?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Creating ML models:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Data lakes enable you to run analytics such as machine learning and big data processing. The amount of data needed for these functions to be effective is massive, and a data lake performs a valuable role in providing a single source for the data that might be required.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Updating data:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" As no data structure is required, data lakes allow you to add new data easily which means that data lakes can be kept up to date. Also, data lakes accommodate both data streaming and batch processing. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Choose your data sources:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" A data lake lets you decide where your data comes from. This means that you can choose the systems that you want to pull data from, whether that’s an internal CRM system or an external marketing automation platform. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data security:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Since all of your data is stored in one centralized place, this means that it’s easier to ensure data security. You can use security tools to encrypt your data, and you can control who has access to it. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Simplify your analysis:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Data lakes also make it easier to analyze your data. This is because it’s all in one place and fully integrated, which makes it easier to perform analysis across different data sources. This can help you make better business decisions. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data governance:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Data governance is a set of rules that determine how data is collected, stored, and used within your organization. Data lakes can help with data governance since everything is in one place.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Disadvantages of Using a Data Lake","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Not everything can go in a data lake","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": While a data lake is a useful place to store all of your data, there are certain types of data that don’t belong there. For example, data that you need to query on a regular basis should be stored in a data warehouse rather than a data lake. This is because data warehouses make it possible to access data more quickly than data lakes. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Lack of structure","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": A data lake is meant to store all of your data, including unstructured, semi-structured, and structured data. While a data lake can make it easier to integrate data across different sources, it can be difficult to make sense of data that comes from a variety of sources. This can be especially problematic if your data lake doesn’t have a clear schema that allows you to organize and understand the data.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Disorganized data:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Since data stored in data lakes is raw and unstructured it can end up being hard to find the relevant data you need at the moment. It's important to set up the necessary catalogs that will help you to find the data you need. If you don't set up the right mechanisms to regulate your data mechanism. Data will get corrupt and lead to reliability issues. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Slow performance:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" The bigger the data lake the slower the performance. Improper data partitioning makes things worse. ","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is a Data Warehouse?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In today’s digital world, businesses are striving to capture and analyze data in order to make informed decisions. Data Warehouses are a major component of most data analysis strategies. Unlike data lakes, data warehouses have structured and defined schema because the data sources are business applications. Data in a data warehouse is cleaner than data in a Data lake.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"bk62I96bntFrMxSSrrMPE","type":"Asset","createdAt":"2022-12-21T10:32:44.089Z","updatedAt":"2023-01-18T17:19:57.947Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"DataWarehouse diagram","description":"Data Warehouse","file":{"url":"//images.ctfassets.net/lm7q36eaz111/bk62I96bntFrMxSSrrMPE/a59429b4a6b66ed52891b1f0bcc204ee/Frame_229-1.png","details":{"size":32397,"image":{"width":1920,"height":350}},"fileName":"Frame 229-1.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The above diagram shows data flowing from two sources: an operational database and an inventory management platform into the data warehouse. The data from the data warehouse is used for batch reporting and business intelligence","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A data warehouse is a centralized information storage location that contains multiple tables of data from different sources, including relational databases, operational business intelligence systems, and other structured data sources. Data retrieved from a data warehouse is used for:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Analysis","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Reporting ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data mining","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"It’s a critical part of any business intelligence (BI) strategy, providing a centralized location where all information can be accessed, analyzed, and visualized – enabling organizations to make strategic decisions and run reports more efficiently. For this reason, it’s often described as the “central nervous system for BI”.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Since data is collected from various sources this gives you a better level of insight. Data scientists use SQL clients and business intelligence tools to access data warehouse data.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The data warehouse has 3 tiers:","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"1. The top tier that presents analytics results. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"2. The middle tier consists of the engine that analyzes data. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"3. The bottom tier contains the database server. SSD drives are used to store data that is accessed frequently. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A data warehouse has multiple databases that have defined schemas and organized tables & columns. This means that you can also add integer or string descriptions.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data warehouses are an effective means of storing and analyzing large quantities of data. They are also a great way for businesses to streamline their data processes, cut costs, and optimize ROI. You will learn more about the benefits of a data warehouse in the next section.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"What Are the Benefits of Using a Data Warehouse?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Here are the benefits of using a data warehouse:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A data warehouse helps you to get data quickly which will help you to create business insights that aid decision-making. A data warehouse is a highly specialized storage platform that’s designed to store and organize data to be easily accessible, searchable, and usable across an organization. ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A data warehouse is well organized to help you compare and analyze relationships between different ideas and concepts. This helps you to plot and understand business trends. Data provides insights into your customers and their behaviors, your business and its operations, and your future goals – but for it to be truly effective, it needs to be accessible and organized in a central location.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Some modern data warehouses, such as the ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.sap.com/products/technology-platform/data-warehouse-cloud.html"},"content":[{"nodeType":"text","value":"SAP data warehouse cloud","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" supports both structured and unstructured data. ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data stored in a data warehouse is stable and non-volatile and uses ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.talend.com/resources/what-is-data-warehouse/"},"content":[{"nodeType":"text","value":"files and folders","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" to store data. Previous data will not be erased when new data gets added.","marks":[],"data":{}}]}]}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Disadvantages of Using a Data Warehouse","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data warehouses are expensive and need more expertise to be maintained. This means that you will have to spend more time finding good talent and implementing cost-optimization measures. In addition, data warehouses collect data from various sources which can be a problem if the data has issues. For example, some fields may accept null values when they are not supposed to. ","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Data Warehouse vs. Data Lake","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Some companies use both data lakes and data warehouses. They store raw data in the data lake and then process it. In the end, the processed data will be moved to the data warehouse. This is typically where a company will require a ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/getting-started-with-data-pipelines-and-data-pipeline-architecture/"},"content":[{"nodeType":"text","value":"data pipeline","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data lakes and data warehouses serve different purposes and do not exist to eliminate each other. Your business use case will dictate which one should be your top priority but using them together is the best. Data warehouses are good for processes that are recurring. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Here is a list of factors you should consider when choosing between a data warehouse and a data lake:","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Data and Quality ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data warehouses retrieve data from business applications and operational databases. Since the data found in data warehouses is not raw, its quality is better than that of a data lake. On the other side, data lakes take in raw and unstructured data that is retrieved from relational and non-relational IoT devices. If you want to store and retrieve quality data, then choose a data warehouse otherwise, choose a data lake.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Schema and data analytics ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Since data warehouses contain structured data and well-designed schemas they are used for Business Intelligence tasks and batch reporting. On the other side, data lakes have raw data that is used for predictive analytics and machine learning.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Performance and cost","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data warehouses are expensive when compared to data lakes because they need well-defined architecture, design, and the tools used to build data warehouses. Data lakes are built using tools such as Hadoop, which are built to be low-cost commodity hardware. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data warehouses have fast performance because their data can be retrieved easily since the data is structured. While data lakes are slower when it comes to retrieving data, and they tend to slow down as data gets bigger.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A data lake is a useful place to store all of your data, but it can be difficult to organize and understand all of this data if it’s not clearly structured. It can also be challenging to integrate data across different sources, especially if you want to combine unstructured and semi-structured data. That said, a data lake makes it possible to store all of your data in one place. This can make it easier to perform analytics across different data sources and gain a deeper understanding of your users, product, and business.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"On the other side, a data warehouse enables you to retrieve data faster and fetches data from business applications to give you structured data. This structured data will help you to make quicker decisions. A data warehouse and data lake have their own pros and cons as we have learned previously in this article. Therefore, it is important to know what are your application’s and business needs and use cases. This will help you to know whether you should use a data lake or a data warehouse. But, it would be good if you used them together– a data lake stores unstructured data used for machine learning, while a data warehouse stores structured data that is used for business intelligence.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"And if you want to use both? That's where you'll need to start thinking of how you will move data between them, as well as between the data sources and data sinks. That's where a technology like ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/what-is-apache-kafka/"},"content":[{"nodeType":"text","value":"Apache Kafka ","marks":[],"data":{}}]},{"nodeType":"text","value":"comes in. Conduktor makes Apache Kafka easy, and you can ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"start your free trial today","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3GpYm8e8ScX12WYASfZqde","type":"Entry","createdAt":"2021-11-05T09:29:35.518Z","updatedAt":"2021-11-05T09:29:35.518Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Lorcan","title":"Customer Engineer","linkedIn":"https://www.linkedin.com/in/lorcan-cooke-b5aa77a2/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"14A3qRf2QTdXJx4nDL3rCD","type":"Asset","createdAt":"2021-11-05T09:29:26.913Z","updatedAt":"2022-09-25T07:29:50.969Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Lorcan","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/14A3qRf2QTdXJx4nDL3rCD/79fbd3877631e7e2942459599edb7d6a/lorcan_2x.png","details":{"size":44928,"image":{"width":192,"height":192}},"fileName":"lorcan@2x.png","contentType":"image/png"}}}}},"publishDate":"2022-12-20T12:00+00:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"54MEBgTCU3VoQ3hbXSOszE","type":"Entry","createdAt":"2022-12-08T15:18:32.309Z","updatedAt":"2022-12-20T11:48:36.103Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"What is Data Streaming?","description":"A primer on real-time data streaming, the innovation that empowers the modern internet. Get a high-level overview of how it works and the challenges faced.","noIndex":false}},"slug":"what-is-data-streaming","title":"What is Data Streaming?","navbarDescription":"Get a high level overview of real-time data streaming, how it works, and the challenges faced.","description":"A primer on real-time data streaming, the innovation that empowers the modern internet. Get a high-level overview of how it works and the challenges faced.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"36GQqFS7hdA7PeSYkxlcUF","type":"Asset","createdAt":"2022-12-20T11:48:52.807Z","updatedAt":"2022-12-20T11:48:52.807Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"DataStreaming","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/36GQqFS7hdA7PeSYkxlcUF/589b061360b29c7057b5b5c0336f5075/DataStreaming.png","details":{"size":76433,"image":{"width":2048,"height":1080}},"fileName":"DataStreaming.png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/what-is-apache-kafka/"},"content":[{"nodeType":"text","value":"Apache Kafka","marks":[],"data":{}}]},{"nodeType":"text","value":" is a data streaming technology, designed to handle the needs of modern organizations which rely extensively on real-time data streaming. But what is data streaming? And why is it so important to the functioning of the modern world?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In this article, we will explore the basics of data streaming, looking at how it works, the emergence of real-time data streaming, the challenges it faces, and its relevance to business.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is Data Streaming?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"First, there is a need to clear up a point of confusion. You will see the terms “data streaming” and “streaming data” used, often interchangeably, but the two can have different meanings. We’ll use the following definitions consistently in this article:","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Streaming data is data that is generated in a continuous stream from a data source. An example of streaming data would be web analytics. Modern web applications measure almost every user activity on their site, e.g., button clicks, page views. Streaming data can also be known as ","marks":[],"data":{}},{"nodeType":"text","value":"unbounded data","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":": unbounded data has no boundaries defined in terms of time. The data may have been arriving from the past, continuing today, and is expected to arrive in the future.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data streaming is a data processing paradigm where the input data arrives continuously, and as soon as it arrives, gets processed. The idea behind data streaming is to capture the incoming data in real time, then process it immediately. The result of this immediate analysis can be used to make decisions that affect the system itself or other systems.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Thus, an example of data streaming would involve using a stream of analytics data - let’s say from Google Analytics - and sending it through a stream processing tool like ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/apache-kafka-vs-amazon-kinesis-comparing-across-five-dimensions/"},"content":[{"nodeType":"text","value":"Amazon Kinesis","marks":[],"data":{}}]},{"nodeType":"text","value":", which might then pass it on to a storage medium or to a business intelligence tool. ","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"How does Data Streaming work?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When it comes to describing the right way to perform data streaming, there isn’t a correct answer. There are a number of approaches to data streaming, each of which will have its place depending on the use case and the data being processed. A ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/getting-started-with-data-pipelines-and-data-pipeline-architecture/"},"content":[{"nodeType":"text","value":"data pipeline ","marks":[],"data":{}}]},{"nodeType":"text","value":"designed for real-time streaming may have a huge number of inputs, outputs, and components depending on what function it is performing.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You should also consider that real-time stream processing is not always necessary for processing data. Batch processing (which involves collecting all the information before doing any analysis) and near-real-time processing (which requires less time than real-time) will be fine in many cases.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"They key factors that separate real-time processing from batch or near-real-time are storage and the processing itself.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Batch processing vs. Stream processing","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Batch processing refers to the method of collecting data from various sources and storing it in database management systems (DBMS). Once all the data has been collected, batch processes are run on them to produce results. The results are then stored in another database management system. If there is any change in the original data set, then all of these processes need to be repeated again. In other words, batch processing is an offline process that doesn't require any user interaction.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Stream processing refers to running queries on real-time data streams as soon as they arrive at a DBMS. A query may contain multiple steps that process events one after another until there are no more events left for processing. This means that when one query is completed, another query can start processing new events coming into your system from various sources","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When it comes to choosing between real-time and non-real-time options for data streaming, there are a few key factors to consider:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Time:","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Obviously, the most important factor is whether the data you are dealing with is time-sensitive or not. For example, prices for ride-sharing services like Uber or Lyft need to be determined in real-time based on demand, whereas other forms of travel like air or rail would not need real-time pricing for tickets.","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Continuous vs Discrete:","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There is also the question of the stream itself. If data is being transferred in discrete chunks as opposed to a continuous stream, there is obviously no need for a real-time data pipeline.","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Fault Tolerance:","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"How sensitive is the data to errors in the pipeline? Can it be sent again? Real-time data streams can rarely be retransmitted, and even if they can the data may not be the same as it was originally. If the data is more tolerant to errors and to retransmission, real-time processing may be unnecessary.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Challenges of working with streaming data","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are several challenges that come with working with streaming data, including the need for real-time processing, the need for scalable and fault-tolerant systems, and the need to deal with potentially infinite amounts of data. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The sheer volume of data that must be processed in real-time puts a lot of strain on computational resources. This leads to requirements for the use of powerful and scalable infrastructure to handle the data.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A related challenge is the need to process data as it arrives, in a timely and efficient manner. This requires the use of specialized algorithms and techniques that can quickly analyze and act on the data, without sacrificing accuracy or reliability.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Additionally, streaming data is often incomplete, noisy, and can arrive in a variety of formats, making it difficult to clean and process effectively. This can require the use of advanced data preprocessing techniques to ensure the data is ready for analysis.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As mentioned above, the systems that handle real-time data streams need to be fault tolerant since the data itself is not. If a stream of data fails to reach its target, the data may be lost permanently or any analysis may be corrupted.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Why is data streaming important for business?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data streaming is important for businesses because it allows them to process and analyze large amounts of data in real-time, which can be crucial for making timely and informed decisions. Data streaming is used to analyze and make decisions based on data as it arrives, rather than waiting until all the data has been collected before starting any analysis. As more and more applications become connected to the internet, there's more pressure on companies to process data streams quickly and accurately.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This is especially important for businesses that need to process large amounts of data from a variety of sources, such as those in the finance, retail, and e-commerce industries. Data streaming also enables businesses to create personalized and engaging experiences for their customers, which can help to improve customer satisfaction and loyalty. Overall, data streaming can help businesses to stay competitive and improve their operations by providing them with the timely and actionable insights they need to make better decisions.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data streaming was first introduced in the late 1990s as a way to process information from sensors, like those used in industrial plants or weather stations. But today, it's being used by many companies as a way to analyze large amounts of data coming from social media, emails, and other sources that provide valuable insight into their customers' needs and wants.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Why should a company use Apache Kafka?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As soon as a company has real-time data streaming needs, a streaming platform must be put in place.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka is one of the most popular data streaming processing platforms in the industry today, being used by more than 80% of the Fortune 100 companies. Kafka provides a simple message queue interface on top of its append-only log-structured storage medium. It stores a log of events. Data is distributed to multiple nodes. Kafka is highly scalable and fault-tolerant to node loss.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka has been deployed in sizes ranging from just one node to thousands of nodes. It is used extensively in production workloads in companies such as Netflix, Apple, Uber, Airbnb, in addition to LinkedIn.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The use cases of Apache Kafka are many. These include stream processing for different business applications. Apache Kafka makes up the storage mechanism for some of the prominent stream processing frameworks, e.g., Apache Flink or Samza.","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Messaging systems","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Activity Tracking","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Gather metrics from many different locations, for example, IoT devices","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Application logs analysis","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"De-coupling of system dependencies","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Integration with Big Data technologies like Spark, Flink, Storm, and Hadoop.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Event-sourcing store","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Nonetheless, Kafka is not designed for every possible real-time data streaming use case, and there are situations where other technologies would work better, such as an embedded real-time technology for IoT, or where a work queue is necessary.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"How to make Kafka pipelines simple","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka is one of the most widely used technologies for data streaming, but many companies continue to struggle with its complexity and a lack of expertise. That’s why we created Conduktor Platform, the best way to handle everything Apache Kafka-related. Conduktor make it simple to take control of every aspect of Kafka development, providing a simple user interface to manage, test, monitor, and optimize your Kafka data streaming pipelines. There’s even a pre-built demo environment to enable you to jump straight in and see how easy Kafka can be: ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"https://www.conduktor.io/get-started/","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":". Give it a try today.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Summary","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This article has scratched the surface of data streaming, providing a glimpse into one of the most important revolutions for modern software. Much of the world around you now depends on real-time data, whether you are transferring money, taking a taxi, streaming a movie, or just browsing the internet. For organizations, developers, and aspiring developers, data streaming is likely to continue to be a focus for growth, with Apache Kafka remaining at the forefront.","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6GhtRQxKIvJvqOJNHbzzNX","type":"Entry","createdAt":"2021-10-08T17:36:57.482Z","updatedAt":"2021-11-05T08:43:44.747Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Matthieu","title":"Software Engineer","twitter":"https://twitter.com/m_baechler","github":"https://github.com/mbaechler/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4XAVdlF6DTNIMk937xc21n","type":"Asset","createdAt":"2021-10-08T17:36:40.309Z","updatedAt":"2022-09-25T07:24:03.530Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Matthieu Baechler","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4XAVdlF6DTNIMk937xc21n/5fc3d6c336db1b8c8dd0ab6ba74ac8d4/matthieu_2x.png","details":{"size":49862,"image":{"width":192,"height":192}},"fileName":"matthieu@2x.png","contentType":"image/png"}}}}},"publishDate":"2022-12-20T12:00+00:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1gekfPGKkZGZkyXgZOdC67","type":"Entry","createdAt":"2022-12-07T13:49:58.190Z","updatedAt":"2023-03-14T20:29:41.376Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"4 New Features of Conduktor Console","description":"Take a look at 4 amazing new features only available through the Conduktor Console","noIndex":false}},"slug":"4-key-new-features-of-conduktor-console","title":"4 key new features of Conduktor Console","navbarDescription":"Take a look at 4 amazing new features only available through the Conduktor Console","description":"What are the coolest things you can do with Conduktor Platform? Aside from testing, monitoring, multi-tenancy, chaos engineering, and all that? It's not just new solutions, as the core Kafka UI offers several amazing new features. Take a look below.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4FYbT19QBDBDHYblpQ8SZ5","type":"Asset","createdAt":"2022-09-23T08:55:17.826Z","updatedAt":"2022-09-25T13:56:52.209Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Data-explore-blog","description":"Data Exploration in Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4FYbT19QBDBDHYblpQ8SZ5/63d7bc1770177d9cb9e5997ef9c51255/blog_-_data-explorer.png","details":{"size":63003,"image":{"width":1200,"height":640}},"fileName":"blog - data-explorer.png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor Platform offers a number of new innovations when it comes to building and maintaining your Apache Kafka ecosystem, but the core of the Platform is still Conduktor Console. The Console is the evolution of Conduktor's original desktop incarnation, providing an intuitive & effective user interface for managing your Kafka topics, producing and consuming data, adding Schema Registry, and more.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We took an in-depth look at the technology of ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/conduktor-console-the-evolution-of-kafka-user-interfaces/"},"content":[{"nodeType":"text","value":"Conduktor Console already","marks":[],"data":{}}]},{"nodeType":"text","value":", so today's article is focusing instead on 4 of the biggest new features introduced by the Console.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"1) Instant Cluster Switching","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7chCWBjsjOw9kAR0o5qyAm","type":"Asset","createdAt":"2022-12-07T11:43:38.341Z","updatedAt":"2022-12-07T11:43:38.341Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"clusterSwitchGif","description":"Instant Cluster Switching","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7chCWBjsjOw9kAR0o5qyAm/c3b6db4802c1171542930d88d581a099/clusterSwitchGif.gif","details":{"size":4682732,"image":{"width":1280,"height":720}},"fileName":"clusterSwitchGif.gif","contentType":"image/gif"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For anyone who has used Conduktor before, the most obvious benefit from switching to the Platform will be the ease with which you can switch between your different Kafka clusters. In the Conduktor Console (or any part of the Conduktor Platform, for that matter), you can switch between your clusters from Amazon MSK, Confluent, Aiven, Instaclustr, open source, or local clusters in a single click.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"2) Message Reprocessing","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"bM7BZKjS0NNwYddhG3sao","type":"Asset","createdAt":"2022-12-07T11:47:07.016Z","updatedAt":"2022-12-07T14:01:33.774Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"reprocessGif","description":"Message Reprocessing","file":{"url":"//images.ctfassets.net/lm7q36eaz111/bM7BZKjS0NNwYddhG3sao/38c0627e33affbd52eb9acd59c46157c/reprocess2.gif","details":{"size":6673633,"image":{"width":1280,"height":720}},"fileName":"reprocess2.gif","contentType":"image/gif"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Message Reprocessing (or republishing) does exactly what it sounds like: you can take a message from a topic and republish it into another topic. It is even possible to edit the contents of the message. Message reprocessing is ideal for dealing with dead letter queues, allowing users to take any rejected message, edit the incorrect section, then reprocess into the correct topic. ","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"3) Revamped User Interface","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor Console looks great, but it is not merely a cosmetic overhaul. The console is able to surface more information and relevant metrics to enable views that simply aren't possible in previous versions of Conduktor. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Topics can now be tagged to enable easy grouping of topics for your different needs:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4GXenbvn44oGE5xUN9jl9R","type":"Asset","createdAt":"2022-12-07T11:52:42.497Z","updatedAt":"2022-12-07T11:52:42.497Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Tags","description":"Organizing Topics","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4GXenbvn44oGE5xUN9jl9R/e356ea0054ccf214b8d245631aed56ac/Tags.png","details":{"size":223461,"image":{"width":1920,"height":1080}},"fileName":"Tags.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The built-in consumer is far more powerful, displaying consumed records much faster along with metrics and graphs:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1iIlWWd8iSdl0cDguyEyyh","type":"Asset","createdAt":"2022-12-07T11:56:30.156Z","updatedAt":"2022-12-07T11:56:30.156Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"ConsumeGif","description":"Consume thousands of messages easily","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1iIlWWd8iSdl0cDguyEyyh/154b68a749731957daaec4d75cc3a57f/ConsumeGif.gif","details":{"size":5756215,"image":{"width":1920,"height":1080}},"fileName":"ConsumeGif.gif","contentType":"image/gif"}}}},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"4) Advanced Filtering + Javascript Queries","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2sHRmC7s4W5VelvpKzKGwy","type":"Asset","createdAt":"2022-12-07T12:03:58.435Z","updatedAt":"2022-12-07T12:03:58.435Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"JSONGif","description":"JSON filters in Conduktor Console","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2sHRmC7s4W5VelvpKzKGwy/2680cfbf33715dc3b52e9b3af8c6e208/JSONGif.gif","details":{"size":993054,"image":{"width":1280,"height":720}},"fileName":"JSONGif.gif","contentType":"image/gif"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The fourth innovation delivered by the Console is improved topic filters and the potential to build JS queries to find exactly what you need. Previous versions of Conduktor did have some capability for logical filters, but they had some limitations. With Javascript, you can build queries to handle anything, with any amount of functions and operators. The Console also provides intelligent format filters, automatically detecting key + value formats when filtering.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor Desktop broke new ground in many ways when it launched, and it is still a great Kafka UI. Conduktor Platform now does everything that Desktop could do and much more. ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Give it a try today ","marks":[],"data":{}}]},{"nodeType":"text","value":"and see for yourself.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"\n\n\n\n","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2SX6zpOndgjWDVop0j59Uz","type":"Entry","createdAt":"2021-10-08T17:41:29.513Z","updatedAt":"2022-09-27T10:40:02.619Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":6,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Stéphane","surname":"Maarek","title":"Chief Evangelist","twitter":"https://twitter.com/StephaneMaarek","github":"https://github.com/simplesteph","linkedIn":"https://www.linkedin.com/in/stephanemaarek/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7y6tdCR7qU2yEWMJmQDS23","type":"Asset","createdAt":"2021-10-08T17:40:54.512Z","updatedAt":"2022-09-25T07:11:34.678Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Stephane Maarek","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7y6tdCR7qU2yEWMJmQDS23/523069a652132097fc9825d7831c0790/stephane-m.png","details":{"size":47223,"image":{"width":192,"height":192}},"fileName":"stephane-m.png","contentType":"image/png"}}}}},"publishDate":"2022-12-09T12:00+00:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"vkurBLCOo9rRkSzIyZw67","type":"Entry","createdAt":"2022-09-23T13:45:06.305Z","updatedAt":"2022-12-07T14:36:16.534Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":5,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Conduktor Console: The Evolution of Kafka User Interfaces","description":"Conduktor Console is the new Platform user interface for Apache Kafka. This blog will take a look at the features of the Console ","noIndex":false}},"slug":"conduktor-console-the-evolution-of-kafka-user-interfaces","title":"Conduktor Console: The Evolution of Kafka User Interfaces","navbarDescription":"Learn everything there is to know about the new Conduktor Console","description":"Conduktor Console is the new user interface for Apache Kafka available in the Conduktor Platform. This blog will take a look at the features of the Console and how it can help your data exploration, Kafka operations, and troubleshooting.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"oU5UUBaj25Lgl8S4J6Nuu","type":"Asset","createdAt":"2022-12-07T14:36:01.423Z","updatedAt":"2022-12-07T14:36:01.423Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"ConsoleDeepdive","description":"Conduktor Console Deep Dive","file":{"url":"//images.ctfassets.net/lm7q36eaz111/oU5UUBaj25Lgl8S4J6Nuu/7eb66192f13976acf8516ebf9f319ec4/ConsoleDeepdive-1.png","details":{"size":98711,"image":{"width":1200,"height":630}},"fileName":"ConsoleDeepdive-1.png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The next in our lineup of solution deep dives is a look at the new Conduktor Console, the successor to Conduktor's DesktopUI for Apache Kafka. The Console's features are divided into three main parts: ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/explorer/"},"content":[{"nodeType":"text","value":"Explorer","marks":[],"data":{}}]},{"nodeType":"text","value":", ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/operations/"},"content":[{"nodeType":"text","value":"Operations","marks":[],"data":{}}]},{"nodeType":"text","value":", and ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/troubleshooting/"},"content":[{"nodeType":"text","value":"Troubleshooting","marks":[],"data":{}}]},{"nodeType":"text","value":", though all three work together to contribute to simplifying Kafka. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor Console was built on the foundations of the original Conduktor desktop application but brings a number of new features and improvements. Let's take a look.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Why do you need Conduktor Console?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If you have used Apache Kafka at any point, this question will be a bit redundant. The need for an easier way to interact with Kafka is obvious from the moment you start. Kafka is well named, for it presents an impassable bureaucracy to new users. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let's do a simple comparison shot against the CLI. In this clip, we are creating a topic, producing a message with a key and a value, then consuming that message:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"OiBbtfjeRcHQNVHNASRYT","type":"Asset","createdAt":"2022-12-07T10:17:26.909Z","updatedAt":"2022-12-08T11:58:40.740Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"ConsolevsCLI","file":{"url":"//videos.ctfassets.net/lm7q36eaz111/OiBbtfjeRcHQNVHNASRYT/e29805675ed952543131791780056731/ConsolevsCLIv4.mp4","details":{"size":5474315},"fileName":"ConsolevsCLIv4.mp4","contentType":"video/mp4"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Of course, this is a very simple example of what you might do with Kafka, but the difference is still vast. Perhaps it is a bit unfair - a Kafka expert could be a lot faster with the CLI; we didn't use terminal history and we made a spelling mistake on purpose. However, ","marks":[],"data":{}},{"nodeType":"text","value":"the Conduktor Console will be that simple for everyone, every time, no experience needed","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". And once you get to the more complicated operations, even experts won't be able to keep up.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"The Conduktor Console Difference","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Few organizations stick with the CLI given its difficulty and poor interface. There are a number of paid and open source tools that try to simplify Kafka, but Conduktor has always been first-class. Conduktor Console simply offers more features and covers more use cases than anything else available.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let's look at how Console handles data exploration:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5Gex21uOe68KRqTKCQv2SU","type":"Asset","createdAt":"2022-12-07T10:31:32.836Z","updatedAt":"2022-12-08T11:51:26.892Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"exploreData","file":{"url":"//videos.ctfassets.net/lm7q36eaz111/5Gex21uOe68KRqTKCQv2SU/a43b71cf1820d066eabad889e80b9eaa/exploreData.mp4","details":{"size":3804564},"fileName":"exploreData.mp4","contentType":"video/mp4"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor Console can display all of your topics and consumer groups, as well as the messages that have been consumed in just a few clicks. Moreover, you can quickly query your clusters to find specific information and filter where needed.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We looked at basic operations above, but there is a lot more that Console can do. Setting up Kafka Connectors and managing them is a big difference maker. You can see how easy fixing issues is as well with instant restarts for any Connectors that have issues:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6HrGJcLx1407K9ldKA8mjK","type":"Asset","createdAt":"2022-12-07T10:33:03.380Z","updatedAt":"2022-12-08T11:51:33.443Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"KafkaConnect","file":{"url":"//videos.ctfassets.net/lm7q36eaz111/6HrGJcLx1407K9ldKA8mjK/0e567eeba13e8756f4372eb84a0cb479/Connect.mp4","details":{"size":3049877},"fileName":"Connect.mp4","contentType":"video/mp4"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Two of the biggest new features available in the Console are rapid cluster switching and message reprocessing. Instead of having to reboot every time you want to work on another cluster, with Console a single click takes you there:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"12QL6pEtwfJSf21RZIxRND","type":"Asset","createdAt":"2022-12-07T10:45:40.643Z","updatedAt":"2022-12-08T11:51:39.841Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"clusterSwitching","file":{"url":"//videos.ctfassets.net/lm7q36eaz111/12QL6pEtwfJSf21RZIxRND/1c2cd1337dd3e0a7f438353555231667/clusterSwitching.mp4","details":{"size":1346368},"fileName":"clusterSwitching.mp4","contentType":"video/mp4"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Message reprocessing allows you to take any message from any topic and instantly republish it to another topic. You can even edit the message's key, value, and headers before reprocessing. ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2OqVRfBIb6atORM9gv3oHw","type":"Asset","createdAt":"2022-12-07T10:53:30.000Z","updatedAt":"2022-12-08T11:51:45.608Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Reprocessing","file":{"url":"//videos.ctfassets.net/lm7q36eaz111/2OqVRfBIb6atORM9gv3oHw/82e05f39c36dba85cc7a207709b39500/Republishing.mp4","details":{"size":5741621},"fileName":"Republishing.mp4","contentType":"video/mp4"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"It's not just individual features that make Conduktor Console such a key tool for teams using Kafka. The Console is part of the wider Conduktor Platform, taking care not just of your Kafka user interface but also of testing, monitoring, governance, and data quality. Check out the previous deep dives on:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/conduktor-testing-deep-dive/"},"content":[{"nodeType":"text","value":"End to end Testing","marks":[],"data":{}}]},{"nodeType":"text","value":" for Apache Kafka","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/security-for-apache-kafka-in-conduktor-platform/"},"content":[{"nodeType":"text","value":"Security: RBAC, SSO, and more","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Other Key Features","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Those are the major changes, but there are many other features to the Console:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Open Platform: The Platform is cluster-agnostic, meaning it will work with any Kafka provider as well as local clusters. Confluent, Amazon MSK, Aiven, and all the rest can be accessed through the Console and the wider Conduktor Platform with no barriers.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ecosystem Support: The Conduktor Console fully supports Confluent Schema Registry and Kafka Connect, enabling virtually all integrations that are widely used today.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Consumer Group Controls: The console allows you to view and edit consumer groups in Kafka.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Customizable Views: Users can customize the appearance of their Console to their liking, enabling dark theme and altering elements to find the best solution.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Deployment options: You can deploy as a Docker image in your environment that needs no internet access or you can use our Cloud offering online","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"How Do I Get Started?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You can start using the Conduktor Console with a single click. Just ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"click here to get started ","marks":[],"data":{}}]},{"nodeType":"text","value":"with the free version of the Console. You can opt for Cloud or Docker.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Once sign-up is complete, you will need to setup a cluster in the admin section. The platform supports open-source Kafka as well as all major providers such as Confluent, Amazon MSK, and Aiven. Head to our ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/platform/category/get-started/"},"content":[{"nodeType":"text","value":"Documentation ","marks":[],"data":{}}]},{"nodeType":"text","value":"page for full instructions on how to configure clusters for both the Cloud and Docker versions.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"And that's it. Head to the Console section in your dashboard and you can start creating topics, producing data, and much more.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Summary","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Everything you have seen in this blog post you can try right now by getting started with the Platform. And this is just the beginning of our roadmap. We're going to continue to add new solutions to the overall Platform, and to improve the experience of the core Conduktor Console.","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5au3vTjeNc7Ctl5QELPO4O","type":"Entry","createdAt":"2021-10-08T18:56:16.912Z","updatedAt":"2022-09-25T14:05:25.231Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Jules","title":"Software Engineer","twitter":"https://twitter.com/guizmaii","github":"https://github.com/guizmaii","linkedIn":"https://www.linkedin.com/in/jules-ivanic/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"xAViG6Oc1Cjy9LhsG7dZ3","type":"Asset","createdAt":"2022-09-25T14:05:22.158Z","updatedAt":"2022-09-25T14:05:22.158Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"jules","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/xAViG6Oc1Cjy9LhsG7dZ3/986f217ee81457b0332c54576efed311/jules.png","details":{"size":50487,"image":{"width":192,"height":192}},"fileName":"jules.png","contentType":"image/png"}}}}},"publishDate":"2022-12-08T12:00+01:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"24PKu4LbhA3JFmrgzfpFv","type":"Entry","createdAt":"2022-12-01T13:43:08.261Z","updatedAt":"2022-12-01T13:43:08.261Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Apache Kafka Partition Assignment Strategy","description":"How are topic partitions assigned to a consumer? What happens when a consumer dies? Find out the different strategies you can use with Apache Kafka","noIndex":false}},"slug":"kafka-partition-assignment-strategy","title":"Kafka Partition Assignment Strategy","navbarDescription":"In this article, we’re going to discuss how topic partitions are assigned to a consumer and what happens when a consumer dies","description":"Kafka consumers pull data from topic partitions, but what happens if a consumer dies? How will the partitions be reassigned, and what is the best method of reassignment? This article looks at the partition.assignment.strategy configuration and the different options it enables.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"yChrYsn4sP2Y9rTzM5UVg","type":"Asset","createdAt":"2022-12-01T13:41:49.080Z","updatedAt":"2022-12-01T13:41:49.080Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"KafkaPartition OG","description":"Kafka Partition Assignment Strategy","file":{"url":"//images.ctfassets.net/lm7q36eaz111/yChrYsn4sP2Y9rTzM5UVg/5edb01b9f2e3ff2edb7ee3d8cbe1eb6e/KafkaPartition_OG.png","details":{"size":98889,"image":{"width":1200,"height":630}},"fileName":"KafkaPartition_OG.png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In this article, we’re going to discuss how ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/kafka-topics/#What-are-Kafka-Partitions?-1"},"content":[{"nodeType":"text","value":"partitions ","marks":[],"data":{}}]},{"nodeType":"text","value":"are assigned to a consumer and what happens when a consumer dies. We shall discuss how we can distribute the partitions among available consumers for better usability and also which strategy will give the better throughput and availability. Once we understand all the assignment strategies, we can decide which one to set for our use case.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When a consumer dies, all of its partitions get assigned to other available consumers and that rebalance of partitions impacts the throughput, availability, and latency of the application. Using the right partition assignment setting for our use case can significantly minimize the impact.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Partition.assignment.strategy","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When a new consumer group is created a consumer group coordinator is elected. The coordinator lives on a broker assigned by a hash() of the consumer group name. This coordinator is responsible for various tasks, including partition assignment, offset commits, and deciding when a consumer should be marked “failed”. Consumers in a consumer group share ownership of the partitions in the topics they subscribe to. When we add a new consumer to the group, it starts consuming messages from partitions previously consumed by another consumer. The same thing happens when a consumer shuts down or crashes; it leaves the group, and the partitions it used to consume will be consumed by one of the remaining consumers.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Reassignment of partitions to consumers also happens when the topics the consumer group is consuming are modified, e.g. new partitions are added to the topic.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"PartitionAssignor","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" is the class that decides which partitions will be assigned to which consumer. When creating a new Kafka consumer, we can configure the strategy that will be used to assign the partitions amongst the consumers. We can set it using the configuration ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/documentation/#consumerconfigs_partition.assignment.strategy"},"content":[{"nodeType":"text","value":"partition.assignment.strategy","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":". ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"All the Kafka consumers which belong to the same consumer group must have a single assignment strategy. If a consumer attempts to join a consumer group that has a different assignment strategy, it will end up getting an ","marks":[],"data":{}},{"nodeType":"text","value":"InconsistentGroupProtocolException","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"By default, Kafka has the following assignment strategies.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"RangeAssignor","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The aim of the RangeAssignor strategy is to have a single consumer instance co-host matching partitions from 1 or more topics. This is useful to join records from topics that have the same number of partitions. This strategy will put all the consumers in an order of member_id assigned by the coordinator and after that, the partitions are assigned starting from the first consumer.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1J6YSOXscYvlXZsA7qWYVj","type":"Asset","createdAt":"2022-12-01T10:32:57.995Z","updatedAt":"2022-12-05T17:12:05.367Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"RangeAssignor","description":"RangeAssignor diagram","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1J6YSOXscYvlXZsA7qWYVj/2c7cd287c0e845ccbc0739287544349e/RangeAssignor.png","details":{"size":29251,"image":{"width":1000,"height":800}},"fileName":"RangeAssignor.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We can see that Partitions 0 and 1 are assigned to the same Consumer but Consumer-3 is sitting idle. So if you’re planning to consume from multiple topics and you don’t have co-localized partitions, you should not use the default partitions assignment strategy.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"RoundRobinAssignor","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With RoundRobinAssignor, the partitions are evenly distributed among all the consumers in the Consumer group. As with RangeAssignor, the partitions and consumers are put in lexicographic order before the assignment. The RoundRobin Assignor gives the advantage of effective usage of all the available Consumers and better performance.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3nFZ8NXhCqvsJaBIeVAJo9","type":"Asset","createdAt":"2022-12-01T10:34:42.102Z","updatedAt":"2022-12-05T17:12:22.375Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"RoundRobinAssignor","description":"RoundRobinAssignor diagram","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3nFZ8NXhCqvsJaBIeVAJo9/f6ad67ce5438e32d49a326e3247d7f8d/RoundRobinAssignor1.png","details":{"size":28970,"image":{"width":1000,"height":800}},"fileName":"RoundRobinAssignor1.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Even if RoundRobin provides the advantage of maximizing the number of consumers used, it has one major drawback: it does not attempt to reduce partition movements when the number of consumers changes (i.e. when a rebalance occurs). Let’s try to understand this through the following diagram where Consumer-2 is disconnected:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6zauws0KZo0wOUf5ZaUfJG","type":"Asset","createdAt":"2022-12-01T10:36:09.673Z","updatedAt":"2022-12-05T17:13:03.355Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"RoundRobinAssignor2","description":"RoundRobinAssignor issues diagram","file":{"url":"//images.ctfassets.net/lm7q36eaz111/6zauws0KZo0wOUf5ZaUfJG/ce8313cea2e4e63e0cd6abc4d7c351a5/RoundRobinAssignor2.png","details":{"size":29589,"image":{"width":1000,"height":800}},"fileName":"RoundRobinAssignor2.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Here we can see that once a Consumer gets disconnected, the assignment of Partitions A-1, B-0, and B-1 change. So out of 4, 3 Partitions got re-assigned to available Consumers. This unnecessary partition movement can impact Consumer performance.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"StickyAssignor","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The StickyAssignor overcomes the challenges of RoundRobin Assignor (unnecessary partition movement). It minimizes the Partitions movement along with having an assignment that is as balanced as possible among available Consumers.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"So let us try to understand the situation when the Consumer-2 instance is marked down or failed in the previous example with StickyAssignor:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5C3rhizBS20Fd5LieW4W3k","type":"Asset","createdAt":"2022-12-01T10:38:26.335Z","updatedAt":"2022-12-05T17:12:50.746Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"StickyAssignor","description":"StickyAssignor diagram","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5C3rhizBS20Fd5LieW4W3k/3d4a9b7964a821cd55838d48f8598660/StickyAssignor.png","details":{"size":29434,"image":{"width":1000,"height":800}},"fileName":"StickyAssignor.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In cases where consumers in the same group subscribe to different topics, the assignment achieved by StickyAssignor is more balanced than that of the RoundRobinAssignor. This normally improves throughput, and latency as well as reducing rebalance time.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"CooperativeStickyAssignor","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To understand CooperativeStickyAssignor, we need to first understand how cooperative rebalances work. But before that, we need to understand ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/consumer-incremental-rebalance-and-static-group-membership/"},"content":[{"nodeType":"text","value":"what a rebalance is","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}},{"nodeType":"text","value":" Moving partition ownership from one consumer to another, in case of: ","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"ordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Addition of new Consumer to the Consumer group, ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Removal of Consumer from Consumer group and ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If new Partitions get added to the existing Topic","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":" is called a rebalance","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". We have two kinds of rebalances:","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Eager Rebalance","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"During an Eager rebalance, all consumers stop consuming the data. The current assignment of partitions expires and Consumers need to rejoin the group, then the new partition assignment happens. So during this time, the consumer group remains idle and does not consume any messages or allow offset commits. Eager rebalances have two distinct phases: first, all consumers give up their assigned partitions, and second, they rejoin the group, get new partition assignments, and resume consuming messages and processing.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Cooperative Rebalance","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Cooperative rebalances (also called incremental rebalances) do not stop all the consumer instances from consuming from the partitions; they only affect the small set of partitions that are moved from one consumer to another. So only the specific partitions which need to be reassigned will not be consumed from and the rest of the partitions will work as usual.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The rebalance is done in two or more phases. In the first phase, the leader informs all the consumers that they will lose ownership of some partitions, then the consumer will give up their ownership of these partitions. In the next phase, the coordinator will assign these orphaned partitions to their new Consumers. The Cooperative rebalance may take a few iterations until a stable partition assignment is achieved, but it avoids the unavailability that occurs with the eager approach. This is very important in the significantly large consumer groups where rebalance takes more time.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6lFY294ZGNsWVkKKcrLAFj","type":"Asset","createdAt":"2022-12-01T10:43:12.430Z","updatedAt":"2022-12-01T10:43:12.430Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"CooperativeRebalance","description":"Cooperative Rebalance diagram","file":{"url":"//images.ctfassets.net/lm7q36eaz111/6lFY294ZGNsWVkKKcrLAFj/f265201631e1b3c7e4108a0f47055338/CooperativeRebalance.png","details":{"size":38490,"image":{"width":1257,"height":740}},"fileName":"CooperativeRebalance.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As we can see in the above diagram, when Consumer-C joins the group, one of the partitions from Consumer-A gets revoked for better usage of all the available Consumers. This was the first phase of the cooperative rebalance; the other partitions (P0 and P1) are still consumable. In the next rebalance, P2 gets assigned to the new Consumer and now it is also back in business. So we have seen that a cooperative rebalance is very effective and tries to minimize the effect of rebalancing.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Now finally coming back to CooperativeStickyAssignor: it is identical to the Sticky Assignor, but supports cooperative rebalances in which consumers can continue consuming from the partitions that are not reassigned.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Creating a Custom Partitioner Strategy","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The ","marks":[],"data":{}},{"nodeType":"text","value":"partition.assignment.strategy","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" configuration allows you to choose a partition assignment strategy out of the above three strategies. For most cases, one out of the available three options would be sufficient but for more advanced scenarios, one can implement their own custom assignment strategy.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To implement the custom assignor, one can use the ","marks":[],"data":{}},{"nodeType":"text","value":"PartitionAssignor ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"interface. It has the following 4 methods:","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5psW5LiSnNbP4g2u7Exptl","type":"Entry","createdAt":"2023-01-18T17:28:41.657Z","updatedAt":"2023-01-18T17:28:41.657Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"java","code":"Subscription subscription(Set topics);\n\nMap assign(Cluster metadata, Map subscriptions);\n\nvoid onAssignment(Assignment assignment);\n\nString name();"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The ","marks":[],"data":{}},{"nodeType":"text","value":"subscription","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"() method is invoked on all consumers, which are responsible for creating the Subscription that will be sent to the broker coordinator. A Subscription contains the set of topics that the consumer subscribes to. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Partition assignment will be done using the ","marks":[],"data":{}},{"nodeType":"text","value":"assign","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"() method.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"All the consumers will receive their assignment from the leader and the ","marks":[],"data":{}},{"nodeType":"text","value":"onAssignment","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"() method will be invoked.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Finally, a PartitionAssignor must be assigned to a unique name returned by the method ","marks":[],"data":{}},{"nodeType":"text","value":"name","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"(). Here you can provide any custom name to the assignor.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In this blog, we have discussed all the available Partition assignment strategies in Kafka. We can choose any of the strategies based on the use case and we can also embed our own protocol to customize how partitions are assigned to the group members for complex and different use cases.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The","marks":[],"data":{}},{"nodeType":"text","value":" partition.assignment.strategy ","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"configuration allows you to choose a strategy. The default is ","marks":[],"data":{}},{"nodeType":"text","value":"org.apache.kafka.clients.consumer.RangeAssignor","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":", which implements the Range strategy described above. You can replace it with ","marks":[],"data":{}},{"nodeType":"text","value":"org.apache.kafka.clients.consumer.RoundRobinAssignor","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":", ","marks":[],"data":{}},{"nodeType":"text","value":"org.apache.kafka. clients.consumer.StickyAssignor","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":", or ","marks":[],"data":{}},{"nodeType":"text","value":"org.apache.kafka.clients.consumer. CooperativeStickyAssignor","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":". A more advanced option is to implement your own assignment strategy, in which case ","marks":[],"data":{}},{"nodeType":"text","value":"partition.assignment.strategy","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" should point to the name of the custom class.","marks":[],"data":{}}]}]},"publishDate":"2022-12-01T01:00+00:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2T764UbZN2jXlBPIvjxsZe","type":"Entry","createdAt":"2022-11-24T10:51:46.551Z","updatedAt":"2022-11-24T10:51:46.551Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"What is the Schema Registry and why do you need to use it?","description":"In this article, we look into why Schema Registry is best practice if you are an Apache Kafka user and how to set it up.","noIndex":false}},"slug":"what-is-the-schema-registry-and-why-do-you-need-to-use-it","title":"What is the Schema Registry and why do you need to use it?","navbarDescription":"In this article we look into why Schema Registry is best practice if you are an Apache Kafka user","description":"Engineer Lorcan Cooke looks into what Schema Registry is and why you should use it for your Apache Kafka. In part 3, Lorcan demonstrates how to set up an Aiven Karapace registry with Conduktor Platform","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1RcWpT9H6CB9x1UBpg5gOL","type":"Asset","createdAt":"2022-11-24T10:51:43.456Z","updatedAt":"2022-11-24T10:51:43.456Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"SchemaRegistry OG","description":"What is Schema Registry?","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1RcWpT9H6CB9x1UBpg5gOL/491d9dcf78f27c84a133f8bebc9801b5/SchemeRegistry_OG.png","details":{"size":55970,"image":{"width":1200,"height":630}},"fileName":"SchemeRegistry_OG.png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In this article, we look into why Schema Registry is best practice if you are an Apache Kafka user.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"First, we take a step back on Apache Kafka and how it transfers data using Zero Copy, serialization, and data formats.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Then we discover what a Schema Registry is and how it can secure your Kafka through schema validation.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Finally, we see an example of how to set up an Aiven Karaspace Registry on Conduktor Console to create, manage and update your schemas while keeping compatibility in check so nothing breaks upstream or downstream.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Part 1: What is Apache Kafka? ","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/what-is-apache-kafka-part-1/"},"content":[{"nodeType":"text","value":"Apache Kafka ","marks":[],"data":{}}]},{"nodeType":"text","value":"is an open-source distributed event streaming platform that is composed of a network of machines called brokers. The brokers accept “events” from event producers and retain them based on time or message size, or both, allowing consumers to read and process them asynchronously.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4GxR3TGjCqWfGxiOQhAkCy","type":"Asset","createdAt":"2022-11-24T09:52:00.074Z","updatedAt":"2022-11-24T09:52:00.074Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"figure 1 - Kafka","description":"basic Kafka architecture diagram","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4GxR3TGjCqWfGxiOQhAkCy/2d235700af454af9a0df3718197b444b/image1.png","details":{"size":122291,"image":{"width":1800,"height":594}},"fileName":"image1.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Figure 1:","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" Example Kafka cluster setup with a producer, a topic, and a consumer (1)","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Zero Copy","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"It is important to understand that Kafka at its core operates on what is called “","marks":[],"data":{}},{"nodeType":"text","value":"Zero Copy” ","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"or sometimes the ","marks":[],"data":{}},{"nodeType":"text","value":"“Zero Copy Principle”","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":". This means that Kafka itself does not read the data that it is sending, it only transfers the data in byte format. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This ensures that Kafka has a significant performance improvement versus more traditional approaches for carrying out a great deal of copying between channels. Kafka also ensures no data verification is carried out at the Kafka cluster level. Kafka itself does not know what kind of data is sent or received.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"666AvZm7E7FAceVaz901Gv","type":"Asset","createdAt":"2022-11-24T10:03:21.468Z","updatedAt":"2022-11-24T10:03:21.468Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Zero Copy vs Traditional","description":"Comparing Zero Copy (On the left) to more traditional approaches (On the right)","file":{"url":"//images.ctfassets.net/lm7q36eaz111/666AvZm7E7FAceVaz901Gv/dab01afec5749cc0876abd7a39672685/image11a.png","details":{"size":169234,"image":{"width":1715,"height":650}},"fileName":"image11a.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Figure 2: ","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"Comparing Zero Copy (On the left) to more traditional approaches (On the right) (3)","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Serialization and Deserialization","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As we have seen in figure 1, producers and consumers in Kafka do not communicate with each other directly; data is transferred between them via a topic. Before we can make use of “Zero Copy” we must utilize a process known as serialization and deserialization.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Serialization, carried out by the producer, converts an object into a stream of bytes used for transmission. Kafka then stores these bytes in the topic(s) the message has been sent to. Deserialization, carried out by the consumer, then converts the bytes of arrays into the desired data type, string, integer, Avro, etc. (4) ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"iftpKEnnxLQ4BJIopDhN1","type":"Asset","createdAt":"2022-11-24T10:06:52.885Z","updatedAt":"2022-11-24T10:06:52.885Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Kafka message serialization","description":"Kafka message serialization for a message with a Integer Key and a String Value","file":{"url":"//images.ctfassets.net/lm7q36eaz111/iftpKEnnxLQ4BJIopDhN1/af186bb7457bccde41877a9245a9d7de/image6.png","details":{"size":467396,"image":{"width":1328,"height":1402}},"fileName":"image6.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Figure 3: ","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"Kafka message serialization for a message with an Integer Key and a String Value ","marks":[],"data":{}},{"nodeType":"text","value":"(4)","marks":[{"type":"italic"}],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7UoXj1cJ10didnA57zZ4r","type":"Asset","createdAt":"2022-11-24T10:07:21.253Z","updatedAt":"2023-01-10T13:13:52.106Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Kafka message deserialization","description":"Kafka message deserialization for a message with a Integer Key and a String Value","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7UoXj1cJ10didnA57zZ4r/ace639b97e6da8e6c54b8ee9917a6342/image16.png","details":{"size":449938,"image":{"width":1386,"height":1270}},"fileName":"image16.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Figure 4: ","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"Kafka message deserialization for a message with an Integer Key and a String Value","marks":[],"data":{}},{"nodeType":"text","value":" (5)","marks":[{"type":"italic"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"So what happens if a producer starts sending data to Kafka that your consumer doesn't know how to deserialize (sometimes known as a “poison pill”[6]) or if the data format is changed without changing the consumer? If this were to happen your consumer(s) downstream will start breaking as they can't properly read the messages, which could mean any applications involved will not work correctly. Depending on what the application is doing you could have a serious problem. The issue is more prominent in Kafka clusters which have many topics, i.e. the more topics you have the more potential cases where producers and consumers deviate due to serialization incompatibility. ","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Data formats and Schemas","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In any process involving integration of data, the two main challenges to consider are the data format and the data schema.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The data format is how the data is parsed, i.e. binary, CSV, JSON, AVRO, Protobuf, etc; and the schema that you will use. There can be a multitude of reasons why you choose a certain data format: depending on legacy systems, choices on latency & throughput, and your company's own knowledge and experience.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A data schema is really the structure or shape of how your messages are sent. You could also consider it the “contract” between producers and consumers that the clients involved have agreed to send and receive messages.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Schema Example in JSON format","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4tf4GXOTCv2t9bjyWd83Mp","type":"Entry","createdAt":"2023-01-18T10:02:25.217Z","updatedAt":"2023-01-18T10:02:25.217Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"json","code":"{\n \"id\": \"invoice_01072022_0982098\",\n \"description\": \"Accommodation invoice\" ,\n \"url\": \"https://acmecompany/invoices/invoice_01072022_0982098.pdf\",\n \"customer\": \"Franz Kafka\" ,\n \"company\": \"The Castle\" ,\n \"total\": 120.99,\n}"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Figure 5: Example of a basic schema for an invoice in JSON format","marks":[{"type":"italic"}],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Part 2: What is the Schema Registry?","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Schema Registry is an external process that runs on a server outside of your Kafka cluster. It is essentially a database for the schemas used in your Kafka environment and handles the distribution and synchronization of schemas to the producer and consumer by storing a copy of the schema in its local cache. ","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"How does the Schema Registry operate?","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A Schema Registry maintains a database of schemas that are stored within an internal Kafka topic known as “_schemas”. (This is a single partition topic that is log compacted).","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7uXDKcbu12N5lq6JtZZ8zM","type":"Asset","createdAt":"2022-11-24T10:17:39.923Z","updatedAt":"2022-11-24T10:17:39.923Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Schema Registry","description":"Schema Registry in Apache Kafka","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7uXDKcbu12N5lq6JtZZ8zM/80f098e24061d9d47b33744e6ae065ce/image3.png","details":{"size":29775,"image":{"width":812,"height":308}},"fileName":"image3.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Figure 6: ","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"Schema Registry in Apache Kafka (1)","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With the Schema Registry in place, the producer, before sending the data to Kafka, talks to the Schema Registry first and checks if the schema is available. If it doesn’t find the schema then it registers and caches it in the Schema Registry. Once the producer gets the schema, it will serialize the data with the schema and send it to Kafka in binary format prepended with a unique schema ID. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When the consumer processes this message, it will communicate with the Schema Registry using the schema ID it received from the producer and deserialize it using the same schema. If there is a schema mismatch, the Schema Registry will throw an error letting the producer know that it’s breaking the schema agreement.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Why use a Schema Registry?","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Having a Schema Registry is really best practice when utilizing Kafka to ensure there is an automated way of ensuring data verification, schema evolution, and ability for new consumers to emerge without breaking downstream. Essentially these are the basics of good data governance when using Kafka.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As your use of Kafka develops and the people who use it change, having a Schema Registry ensures:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data verification ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Peace of mind","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"No breaking schemas even as your Kafka and team evolve.","marks":[],"data":{}}]}]}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"What can happen if you don’t use a Schema Registry?","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let's say our team is working with Kafka, we are an online retail company and use a combination of JSON and Avro formats but mostly JSON. Our initial use of Kafka is as a small team of 5, however as Kafka grows to become business critical and spreads with other teams, we have users not familiar with the environment and at the same time we have had one or two employees move jobs and/or teams as sometimes happens, life and so on. In a few months time, someone who has never spoken to and probably never will speak to the original writer of one of the JSON schemas on our topic for Customer Orders makes a change to the input by adding or removing a field. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The topic then breaks, but we don't know what the problem is and it takes half a day to figure out why and where this schema was originally written, then we need to change the compatibility and also update the schema itself, all manually. A lot of hassle and unnecessary problems could have been avoided by using a Schema Registry. ","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"In summary:","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka operates on the Zero Copy Principle and utilizes serialization and deserialization to achieve this.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This ensures efficient data transfer versus other approaches, however Kafka doesn't know what data it is sending or receiving as it is just bytes.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Using a Schema Registry ensures you have data verification and data versioning for your Kafka producers and consumers.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This gives peace of mind that as your team and Kafka environment evolves no schema will break upstream or downstream and ruin your applications.","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Part 3: Setting up a Schema Registry and checking compatibility","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Tutorial","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We have a better understanding of what a Schema Registry is and why we should use it as best practice in Apache Kafka, but how can you manage your schemas and Schema Registry in practice?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To do this we will set up a Kafka cluster on Aiven with a Schema Registry called Karaspace and then use ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Conduktor Platform ","marks":[],"data":{}}]},{"nodeType":"text","value":"to add a new schema, update this schema, and check its compatibility. This can be broken into the following four steps.","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Step 1:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Connecting your Aiven Cluster on Conduktor Platform","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Step 2: ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"Create a topic and a schema on that topic ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Step 3: ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"Send messages through the producer function with that schema.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Step 4:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Make a change to the schema that might happen in real life and check compatibility again.","marks":[],"data":{}}]}]}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Prerequisites","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Before we can carry out the above we need to have some prerequisites completed as shown below.","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"An Aiven account with 3 brokers Kafka cluster and Karaspace Schema Registry set up and running. ","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":" See Aiven ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://developer.aiven.io/docs/products/kafka/getting-started.html"},"content":[{"nodeType":"text","value":"getting started","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" and ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://github.com/aiven/karapace/blob/main/README.rst"},"content":[{"nodeType":"text","value":"Karaspace docs","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" for more information.","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"An account on Conduktor Platform and running. ","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"You can follow the instructions here","marks":[],"data":{}}]},{"nodeType":"text","value":" to get started.","marks":[],"data":{}}]}]}]}]}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"1) Configuring your Kafka Cluster","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We have our Kafka running on Aiven, with the Karaspace Schema Registry and Conduktor Platform running. We now want to configure our Kafka cluster on Conduktor Platform. To do this we need the information from our Aiven Console (figure 7) and then go to the “Clusters” tab of the Admin section on Conduktor Platform to enter this information (figure 8). The cluster name is of your own choosing. ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3pFTXLIQkwU5FoCMmTkibp","type":"Asset","createdAt":"2022-11-24T10:27:14.949Z","updatedAt":"2022-11-24T10:27:14.949Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Aiven cluster configurations","description":"Aiven cluster configurations view","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3pFTXLIQkwU5FoCMmTkibp/148beed91ed702456aa55b9e57c57024/image9.png","details":{"size":444526,"image":{"width":1707,"height":1999}},"fileName":"image9.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Figure 7: Aiven Cluster Configurations","marks":[{"type":"italic"}],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"92KHNjWVxfjnu0bULM1rQ","type":"Asset","createdAt":"2022-11-24T10:28:00.844Z","updatedAt":"2022-11-24T10:28:00.844Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Aiven cluster configuration in Conduktor Platform","description":"Configuring your aiven cluster in Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/92KHNjWVxfjnu0bULM1rQ/c4b636332614d8b2aa39af00b1b4670b/image17.png","details":{"size":452954,"image":{"width":1968,"height":1999}},"fileName":"image17.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Figure 8: Cluster Configuration on Conduktor","marks":[{"type":"italic"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We then enter the bootstrap server for our Aiven Cluster and the advanced settings should be input in the format shown below, making sure the certificate is on one single line. We click save and make our way to Conduktor Console.","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"60X9lsY6kSoLVmVsJEMX1c","type":"Entry","createdAt":"2023-01-18T17:31:54.199Z","updatedAt":"2023-01-18T17:31:54.199Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"terminalBlock"}},"locale":"en-US"},"fields":{"command":"security.protocol=SASL_SSL sasl.mechanism=SCRAM-SHA-513 sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username=\"\" password=\"\" ssl.truststore.type=PEM sss.truststore.certificates=-----BEGIN CERTIFICATE----- -----END CERTIFICATE-----"}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"2) Creating a topic and adding a schema","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Time to add a topic and create a schema on our Kafka cluster. On Conduktor Platform we go to Console and click “Create Topic” to generate a new topic called invoice-orders.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1blo58Bj3GwElaI7sM4IMS","type":"Asset","createdAt":"2022-11-24T10:34:55.174Z","updatedAt":"2022-11-24T10:34:55.174Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Create a topic","description":"Creating a topic on Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1blo58Bj3GwElaI7sM4IMS/d9a233085c4daadc2f92fe775d61e084/image2a.png","details":{"size":889681,"image":{"width":3964,"height":1999}},"fileName":"image2a.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Figure 9: Creating a topic on Conduktor Platform","marks":[{"type":"italic"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"After this, we navigate to the “Schema Registry” tab and click ","marks":[],"data":{}},{"nodeType":"text","value":"“Create new schema”","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" in the top right corner. We navigate to the “Schema Registry” tab and click ","marks":[],"data":{}},{"nodeType":"text","value":"“Create schema”","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" in the top right corner. Copy the schema shown below and import it into our schema registry.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7KOxEu4Z0Hz3X1OLfFjS1","type":"Asset","createdAt":"2022-11-24T10:42:15.597Z","updatedAt":"2022-11-24T10:42:15.597Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Creating Schema","description":"Creating a Schema on Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7KOxEu4Z0Hz3X1OLfFjS1/825fca22e426f190db368a6250bdcf93/image10a.png","details":{"size":499876,"image":{"width":3991,"height":1816}},"fileName":"image10a.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Figure 10: Creating a schema on Conduktor Platform","marks":[{"type":"italic"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The schema we will be using is a simplified JSON version of one being used for invoice processing.","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5Wd9YM1UiI0wP8TkqBF0Ne","type":"Entry","createdAt":"2023-01-18T17:32:34.143Z","updatedAt":"2023-01-18T17:32:34.143Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"json","code":"{\n \"type\":\"record\",\n \"name\":\"Invoice\",\n \"namespace\":\"com.invoice.example\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"type\":\"string\"\n },\n {\n \"name\":\"description\",\n \"type\":\"string\"\n },\n {\n \"name\":\"customer\",\n \"type\":\"string\"\n },\n {\n \"name\":\"company\",\n \"type\":\"string\"\n },\n {\n \"name\":\"total\",\n \"type\":\"double\"\n }\n ]\n}"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"While the schema vocabulary is out of the scope for this blog post, you can see six properties (key-value pairs) specified that describe an example invoice. Additionally, we make sure that all the objects are required, i.e. a message without all properties should be deemed as invalid. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Below is an example of a message according to this schema:","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7hAmSQSdJ0scPMD2LHiLnG","type":"Entry","createdAt":"2023-01-18T17:33:41.298Z","updatedAt":"2023-01-18T17:33:41.298Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"json","code":"{\n \"Invoice No.\": \"01072022_0982098\",\n \"description\": \"Accommodation invoice\" ,\n \"customer\": \"Franz Kafka\" ,\n \"company\": \"The Castle\" ,\n \"total\": 120.99,\n}"}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"3) Produce messages according to the schema with a producer","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We now want to produce messages according to the schema we have created. Conduktor Platform allows you to produce messages which can infer the data types of the schema concerned. Going back to the “invoice-orders” topic. We can produce one message or a flow of messages and have it end after a number of messages or after a period of time, as shown in figure 11.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This can help for testing in development or to trigger a specific message or event when it hasn't reached the topic.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"71ZXLPBtIixPFjIGJ4f22z","type":"Asset","createdAt":"2022-11-24T10:45:05.077Z","updatedAt":"2022-11-24T10:45:05.077Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Produce Messages","description":"Producing messages on “invoice-orders” topic","file":{"url":"//images.ctfassets.net/lm7q36eaz111/71ZXLPBtIixPFjIGJ4f22z/cdd0d94d75d7e12fd0af010fd8f405d8/image8a.png","details":{"size":428870,"image":{"width":3999,"height":1313}},"fileName":"image8a.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Figure 11: Producing messages on “invoice-orders” topic","marks":[{"type":"italic"}],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"4) Update the schema and check compatibility","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In real life, our use of Kafka and the team that uses Kafka is going to change. Let's imagine we need to remove a field on this schema, whether that is because we want to cut down the information being sent, we don't trust our source of that particular information or it's just not required anymore. ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5yE3D1tvfW17UC4KKVOFj7","type":"Asset","createdAt":"2022-11-24T10:47:51.773Z","updatedAt":"2022-11-24T10:47:51.773Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Updating Schema","description":"Updating Schema and checking compatibility","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5yE3D1tvfW17UC4KKVOFj7/c08501070f050e5707cff14ddf952580/image7a.png","details":{"size":391088,"image":{"width":3997,"height":1313}},"fileName":"image7a.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Figure 12: Updating Schema and checking compatibility","marks":[{"type":"italic"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In this case, we have removed the “Description” field, which is a string. To do that we will click “Update Schema” and remove this field, and before making this change we will “Check compatibility”. This lets us understand if this change will break anything upstream or downstream on our applications.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"52T8GhpWwQTTzwpM4Gkp2X","type":"Asset","createdAt":"2022-11-24T10:48:25.951Z","updatedAt":"2022-11-24T10:48:25.951Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"locale":"en-US"},"fields":{"title":"Updating the Schema","description":"Updating the Schema once it is compatible","file":{"url":"//images.ctfassets.net/lm7q36eaz111/52T8GhpWwQTTzwpM4Gkp2X/b22accc5f4697ce71242714b64f6bd70/image5.png","details":{"size":234630,"image":{"width":1999,"height":1313}},"fileName":"image5.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Figure 13: Updating the Schema once it is compatible","marks":[{"type":"italic"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As can be seen after checking the compatibility, our update is not going to make any breaking changes upstream or downstream so we click update and are able to make our required change.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Final words","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Using a Schema Registry is best practice to control what is produced and consumed within your Kafka cluster. Each schema defines the data format of the topics it is linked to, allowing you to make fields mandatory or to force the type of field.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"It improves the quality of your Kafka data by standardizing it, and ensures consistency and viability over time. Indeed, the schemas have a version and can be updated. With Conduktor, you can check the compatibility between the previous and the new schema, in order to make sure that the modification does not impact your cluster, nor your producers and consumers.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Check out Conduktor Platform ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"here","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" and Aiven Console ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://developer.aiven.io/docs/products/kafka/getting-started.html"},"content":[{"nodeType":"text","value":"here","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Bibliography","marks":[],"data":{}}]},{"nodeType":"ordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/what-is-apache-kafka-part-2/"},"content":[{"nodeType":"text","value":"https://www.conduktor.io/kafka/what-is-apache-kafka-part-2/","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://developer.ibm.com/articles/j-zerocopy/"},"content":[{"nodeType":"text","value":"https://developer.ibm.com/articles/j-zerocopy/","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://developpaper.com/what-is-the-so-called-zero-copy-technology-in-kafka/"},"content":[{"nodeType":"text","value":"https://developpaper.com/what-is-the-so-called-zero-copy-technology-in-kafka/","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/kafka-producers/"},"content":[{"nodeType":"text","value":"https://www.conduktor.io/kafka/kafka-producers","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/kafka-consumers/"},"content":[{"nodeType":"text","value":"https://www.conduktor.io/kafka/kafka-consumers","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.10xbanking.com/engineeringblog/bulletproofing-kafka-and-avoiding-the-poison-pill"},"content":[{"nodeType":"text","value":"https://www.10xbanking.com/engineeringblog/bulletproofing-kafka-and-avoiding-the-poison-pill","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://medium.com/@stephane.maarek/introduction-to-schemas-in-apache-kafka-with-the-confluent-schema-registry-3bf55e401321"},"content":[{"nodeType":"text","value":"https://medium.com/@stephane.maarek/introduction-to-schemas-in-apache-kafka-with-the-confluent-schema-registry-3bf55e401321","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://data-flair.training/blogs/kafka-schema-registry/"},"content":[{"nodeType":"text","value":"https://data-flair.training/blogs/kafka-schema-registry/","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://medium.com/slalom-technology/introduction-to-schema-registry-in-kafka-915ccf06b902"},"content":[{"nodeType":"text","value":"https://medium.com/slalom-technology/introduction-to-schema-registry-in-kafka-915ccf06b902","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://medium.com/@sunny_81705/what-makes-apache-kafka-so-fast-71b477dcbf0"},"content":[{"nodeType":"text","value":"https://medium.com/@sunny_81705/what-makes-apache-kafka-so-fast-71b477dcbf","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://andriymz.github.io/kafka/kafka-disk-write-performance/#"},"content":[{"nodeType":"text","value":"https://andriymz.github.io/kafka/kafka-disk-write-performance/#","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://livebook.manning.com/book/kafka-in-action/chapter-11/13"},"content":[{"nodeType":"text","value":"https://livebook.manning.com/book/kafka-in-action/chapter-11/13","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://medium.com/@sunny_81705/what-makes-apache-kafka-so-fast-71b477dcbf0"},"content":[{"nodeType":"text","value":"https://medium.com/@sunny_81705/what-makes-apache-kafka-so-fast-71b477dcbf0","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://aseigneurin.github.io/2018/08/02/kafka-tutorial-4-avro-and-schema-registry.html"},"content":[{"nodeType":"text","value":"https://aseigneurin.github.io/2018/08/02/kafka-tutorial-4-avro-and-schema-registry.html","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3GpYm8e8ScX12WYASfZqde","type":"Entry","createdAt":"2021-11-05T09:29:35.518Z","updatedAt":"2021-11-05T09:29:35.518Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Lorcan","title":"Customer Engineer","linkedIn":"https://www.linkedin.com/in/lorcan-cooke-b5aa77a2/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"14A3qRf2QTdXJx4nDL3rCD","type":"Asset","createdAt":"2021-11-05T09:29:26.913Z","updatedAt":"2022-09-25T07:29:50.969Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Lorcan","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/14A3qRf2QTdXJx4nDL3rCD/79fbd3877631e7e2942459599edb7d6a/lorcan_2x.png","details":{"size":44928,"image":{"width":192,"height":192}},"fileName":"lorcan@2x.png","contentType":"image/png"}}}}},"publishDate":"2022-11-24T14:00+00:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"wgfms7SnYpjH66gyAIXrw","type":"Entry","createdAt":"2022-11-10T16:19:47.079Z","updatedAt":"2022-11-11T10:22:16.534Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Getting Started with Data Pipelines and Data Pipeline Architecture","description":"In this blog, we’ll look at data pipelines and the various potential data pipeline architectures.","noIndex":false}},"slug":"getting-started-with-data-pipelines-and-data-pipeline-architecture","title":"Getting Started with Data Pipelines and Data Pipeline Architecture","navbarDescription":"In this blog, we will look at data pipelines and the various potential pipeline architectures.","description":"In this blog, we’ll look at data pipelines and the various potential pipeline architectures. We’ll also explore the technologies behind data pipelines, their features, and how they are constructed.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1FLcoH9fnErFdUNUyoCawI","type":"Asset","createdAt":"2022-11-10T16:19:39.884Z","updatedAt":"2022-11-11T10:56:03.429Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":6,"locale":"en-US"},"fields":{"title":"data-pipelines","description":"data pipelines image","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1FLcoH9fnErFdUNUyoCawI/fd62dea41097ef0e6c45ad3ab2b708da/zzz__1_.png","details":{"size":50544,"image":{"width":2048,"height":960}},"fileName":"zzz (1).png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The movement of data is one of the fundamentals of the modern internet. Technologies like ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/"},"content":[{"nodeType":"text","value":"Apache Kafka ","marks":[],"data":{}}]},{"nodeType":"text","value":"fit into wider architectures called data pipelines, which describe how data moves from one place to another. In this blog, we’ll look at data pipelines and the various potential pipeline architectures. We’ll also explore the technologies behind data pipelines, their features, and how they are constructed.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is a Data Pipeline? ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A data pipeline is a continuous process that moves data from one system to another, while transforming the data into actionable formats. A common analogy for describing the concept of a data pipeline is that of water flowing through pipes. In this analogy, the source represents where water comes from, while the transformations represent the different filters needed to purify it before reaching its destination.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data pipelines are used for many different purposes, including:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data acquisition — getting data from data sources (e.g., databases) into the pipeline.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data transformation — transforming unstructured data into structured data or vice versa, or transforming one type of structured data into another type of structured data (e.g., from JSON to XML).","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data cleaning — removing duplicates and other noisy information from the dataset.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data enrichment — adding additional information to the dataset (e.g., adding names and addresses to address records).","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data aggregation — grouping records together so that you can perform analysis on aggregated groups rather than individual records (e.g., calculating average sales by city).","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data storage — storing incomplete datasets in some way that allows them to be accessed later or shared with others who might be interested in them (e.g., saving results in a database or sending them out over email).","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is Data Pipeline Architecture?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A data pipeline architecture is simply the description of what a data pipeline does and how it works. This can be in the form of a logical architecture, which describes the processes and concepts applied to a pipeline. For example, data is moved from a source to a database, processing occurs on the database, logs are captured using Change Data Capture (CDC), and data is delivered to a Business Intelligence (BI) tool. The alternative is a systems architecture which is a literal representation of the tools and platforms used in a pipeline, for example, the architecture is built with Elasticsearch & Apache Kafka and runs on AWS.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Thus if we were to sketch out the basics of creating a data pipeline, we could have a 4-step process:","marks":[],"data":{}}]},{"nodeType":"ordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A need to move data is identified within an organization","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A logical data architecture is devised to solve the problem","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A physical data architecture is built upon the logical data architecture, describing the tools needed.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The data pipeline is implemented by the organization.","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Of course, things are rarely so simple in the real world. Even if you proceed in such an ordered manner, the decisions you will need to make at each stage will be numerous and challenging, involving many different teams.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"An Example Data Pipeline Architecture","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Here’s what a data pipeline architecture might look like, showing the Change Data Capture (CDC) system for Homepoint, a US-based mortgage broker:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2HM2Xv7lAwnDQDamD0DgMk","type":"Asset","createdAt":"2022-08-02T15:34:56.723Z","updatedAt":"2022-11-11T10:07:05.270Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"homepoint-case-study1","description":"Homepoint's Kafka-based system for automated redisclosure","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2HM2Xv7lAwnDQDamD0DgMk/cf24408f99ce4481920bead5718450cf/diagram__1_.png","details":{"size":32440,"image":{"width":2992,"height":1674}},"fileName":"diagram (1).png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This pipeline illustrates how Homepoint transmits information about mortgage redisclosures (basically changes to a loan) automatically. The start of the pipeline is their Loan origination system database, with the end target being another database that controls dashboards for end users. Apache Kafka handles the movement of data throughout the pipeline.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/case-studies/homepoint/"},"content":[{"nodeType":"text","value":"Read more about Homepoint’s Kafka-based data pipeline here","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Components of a Data Pipeline","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let’s return to the 4-step process we mentioned above. After you have identified the problem, you should create a logical data architecture to lay out how you could solve it. What would this look like? ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There isn’t a set of guidelines or rules when it comes to the design of a data pipeline, and you’ll find a variety of terms used across different resources. We can’t provide a definitive list of components, but the following are common and will likely form the core of any pipeline.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Data Sources/Data Collection","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Every pipeline will depend on there being some data collected somewhere. There are thousands of potential sources of data - any and every computer device, to name a few. Some typical business examples will include website events, pricing data, financial transactions, and user interactions.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Ingestion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The data that you have in your sources needs to be moved somewhere in order to accomplish anything with it - otherwise you just have a database, and not a pipeline. This is where a messaging system like Apache Kafka comes in, connecting to data sources and providing fault-tolerant, highly-scalable message queues on top of its append-only log-structured storage medium.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Typically, the end target for data ingestion will be object storage like Google Cloud Storage or Amazon S3, or databases like PostgreSQL or MySQL. ","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Transformation","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Once your data is collected and ingested into your storage - typically referred to as a “data lake” - it needs to be processed into something usable. While some data profiling may take place in the initial collection of data, you will still be working with raw data. This data will need to be cleaned and altered for it to be ready for further analysis. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Transformation may involve a number of different steps, from compression and aggregation to filtering and combination. ","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Consumption","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Your transformed data is now ready to be consumed, which will involve transferring it to suitable targets. Data warehouses may store cleaned data ready for BI or analytics at a later date, but data can also go directly to BI and analytics tools, and potentially directly to applications that require real-time data.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Data Pipeline Architecture Patterns","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Moving beyond the core building blocks of a data pipeline architecture, there are a handful of common patterns that are used frequently.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"ETL Data Pipelines","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"One of the most common forms of a data pipeline is an ETL (extract-transform-load) pipeline, which is used for batch data workloads, like data replication across a pipeline. An ETL pipeline could synchronize source databases with downstream data systems such as caches, search indexes, etc. These pipelines are scheduled to run at weekly, daily, or hourly intervals, typically performing batch workloads. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In an ETL pipeline, data is ‘extracted’ from a data source, ‘transformed’ into an appropriate form, and ‘loaded’ into the target destination.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"ELT Data Pipelines","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There is an alternative to ETL, known as ELT. ELT stands for Extract-Load-Transform. Essentially, the difference between ETL and ELT is just order; with ELT pipelines, data is loaded into a target system to be transformed later, instead of being transformed prior to loading. So, what’s the point? Why choose one over the other?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"ELT is the newer implementation, and its primary advantage is speed - you can implement an ELT pipeline easily, just dumping unstructured data into your storage without having to worry. However, one of the reasons why ELT was not used in favor of ETL was related to storage as well - storage cost. In the past, storage was a lot more expensive, and it is only with the boom in low cost storage, particularly cloud, that ELT has become viable. ETL helped keep costs down as your transformed data was well-defined with no excess.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"That is not to say that one approach is definitely better than the other. The ELT approach can give rise to tech debt as unmodelled data is loaded into your storage and query times grow until you eventually are forced to remodel it properly. Taking the time to define your needs ahead of time helps to prevent this kind of issue.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Data Warehouse Pipelines with ETL","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The traditional method of building a data pipeline was to build around a single data warehouse, using ETL to move data from sources to destinations. A warehouse, unlike a data lake, takes in only structured data that requires a consistent schema to implement. This is one of the main drawbacks of this implementation as modern data flows are increasingly unstructured and complex.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Data Lake Pipelines with ETL","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Centralized data lakes are an important component of modern data architecture. They provide a single, unified place where all of your organization's data can be stored and accessed. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In this pattern, you would use an ETL tool to ingest data into the centralized data lake. For example, if your organization has many different sources of data spread across multiple systems, you could use a tool like Talend or DataStage to pull the data from each source and load it into the central repository. Once there, you could then query, analyze and visualize the data using tools such as Tableau or Alteryx.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"This pattern has a number of advantages over other options:","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"It's relatively easy to set up because there is only one source of truth for your organization's data. This means that anyone who wants to access or transform the data will only have to deal with one system instead of several.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"It provides a consistent view of all your organization's relevant information whether they're looking at historical or current records. This allows users to make better decisions because they aren't limited by incomplete information sets or missing pieces of context that might otherwise cloud their judgement.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Data Mesh Pipelines","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data mesh pipelines are similar to centralized data lake pipelines but use multiple data stores instead of just one. Each component of the pipeline may be hosted on its own server, with each component connected to all others by a network of pipes (hence the name “data mesh”). The end result is that each component has access to all of your enterprise’s data sources, including cloud services like Amazon S3 and Salesforce PaaS.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The advantage of this architecture over centralized pipelines is that you can scale up by adding more microservices instead of increasing parallelism.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Batch vs Streaming vs Lambda Architecture","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"ETL pipelines are a form of batch data pipelines, in which data processes are scheduled to occur at intervals. However, often you will need real-time pipelines that can enable quicker action than batching. For example, above we described a common ETL use case, in which changes to data are replicated across all the systems in a pipeline. Change Data Capture is a real time solution for data replication, which enables downstream consumers to act upon changes quickly.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Real time data pipelines are known as streaming data pipelines. Stream processing technologies like Apache Flink, Apache Storm, and Apache Samza are often deployed in streaming pipelines in order to handle real-time data transformation. Apache Kafka is sometimes referred to as a stream processing technology, but this is incorrect. You can use the Kafka Streams library for stream processing with Kafka, but Kafka itself was not designed for it.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Lambda Architecture is essentially a combination of batch and stream processing, providing a hybrid approach. This is often deployed for big data use cases where you will want to perform batch processing of large, indexed data sets but still have access to real time views of recent, unindexed data.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Data pipeline technologies and techniques","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In every stage of a data pipeline, there are corresponding technologies that handle the processes required for the pipeline to function. Planning a logical data pipeline architecture still leaves you with nothing to work with. You will need to create a physical data pipeline architecture that eventually leads to the implementation of your pipeline.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We have already mentioned a number of potential technologies above, such as Apache Kafka, Apache Flink, or MySQL, but we have barely covered a fraction of the different technologies that are in use all around the world. A full accounting of every different technology would be many times longer than this entire article. And we have only briefly discussed the techniques that might be used in a data pipeline, like ETL and CDC.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When it comes to choosing what technologies and techniques you will use in a pipeline, you don’t necessarily need to know every single option out there. Rather, what is important to understand is how you will allocate the resources you have available. In every case, there are options to do-it-yourself, to turn to open source solutions, or to pay for SaaS providers. Naturally, each option has different costs associated. You are going to be trading off employee time against company money depending on how much work you take on yourself vs how much you pay other companies to do it for you.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Each technology you might deploy will also have a web of connected applications, libraries, and tools that you might need. In every case, you will also need to decide how much investment your individual data pipeline technologies require. ","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"An Implementation Example","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let us use Kafka as an example. Kafka is very widely used as an event-driven messaging system in many modern data pipelines. It is not the only solution to this use case - you could use alternatives like ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/comparing-apache-kafka-activemq-and-rabbitmq/"},"content":[{"nodeType":"text","value":"RabbitMQ, ActiveMQ","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", or ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/comparing-apache-kafka-apache-pulsar/"},"content":[{"nodeType":"text","value":"Apache Pulsar","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":". However, if we assume you are using Kafka, the first decision you are likely to face is whether you will provision your own Kafka clusters using open source Kafka, or turn to a managed Kafka provider like Confluent, Amazon MSK, or Aiven.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"At the same time, you will need to decide on the scope of your Kafka implementation. As we mentioned above, Kafka can take on stream processing tasks by making use of Kafka Streams. Another library, kSQL DB, allows you to perform SQL-like operations on data in Apache Kafka. If you wanted, you could use Kafka for transformations in your data pipeline and some companies even deploy it as their single source of truth, using it practically as a database. There is nothing wrong with this approach, but there will be a sacrifice from not using specialized technologies for these use cases.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Once you’ve decided on these aspects of your Kafka deployment, you will then need to think about tooling. You can do everything in Kafka through the command line, but it is very clunky and practically no organizations rely on it long term. Most instead rely on Kafka user interface tools. You’ll also need to think about your development environment. How will you monitor your Kafka? How are you going to test your Kafka pipeline? Will you need encryption on the data going through it? Who’s going to have visibility, and what will their roles be?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Fortunately, there is an all-in-one solution for all of these issues - ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/explorer/"},"content":[{"nodeType":"text","value":"Conduktor Platform","marks":[],"data":{}}]},{"nodeType":"text","value":". Conduktor Platform provides an intuitive and powerful user interface for handling all of the day-to-day operations that Apache Kafka requires. But it also goes far beyond that, providing a visual testing interface that can handle all Kafka testing needs end-to-end. Monitoring and alerting are built-in and integrate seamlessly with other monitoring solutions. End-to-end encryption can be delivered with a single API call. And those are just a handful of the capabilities available with the Platform. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You can start using the Platform for free with no time limit - just head to our ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Get Started ","marks":[],"data":{}}]},{"nodeType":"text","value":"page. If you’re considering using Kafka in your data pipeline, why not see how much simpler we can make it for you?","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data pipelines are the backbone of modern IT. Practically every application, website, and technology is going to involve data pipelines at some point. However, deciding on an architecture and executing it is far from easy. There are a huge number of different approaches, different use cases, and different technologies that can be applicable. This blog has only scratched the surface of everything there is to know. If you do eventually decide on Apache Kafka as one of your pipeline technologies, ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Conduktor is here to help","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"oH08Uc4XnkIX8LuvAXRpF","type":"Entry","createdAt":"2021-11-05T10:31:13.642Z","updatedAt":"2022-09-25T11:45:59.527Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Roch","title":"Software Engineer","github":"https://github.com/rokh-conduktor","linkedIn":"https://www.linkedin.com/in/rdardie/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"38BbVz2gNg5sHDusF2emge","type":"Asset","createdAt":"2021-11-05T10:31:09.458Z","updatedAt":"2022-09-25T14:02:06.804Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Roch","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/38BbVz2gNg5sHDusF2emge/4548b1829836d06e47702229f10bf7d2/roch.png","details":{"size":53264,"image":{"width":192,"height":192}},"fileName":"roch.png","contentType":"image/png"}}}}},"publishDate":"2022-11-10T17:00+00:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1rftoKf0a7Q7yaP56Ccq2l","type":"Entry","createdAt":"2022-09-23T13:43:24.591Z","updatedAt":"2022-09-25T13:58:42.576Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Conduktor Testing - Deep Dive","description":"Conduktor Testing provides a simple, visual interface for testing applications that incorporate Apache Kafka. Find out more in our deep dive","noIndex":false}},"slug":"conduktor-testing-deep-dive","title":"Conduktor Testing - Deep Dive","navbarDescription":"Conduktor Testing provides a simple interface for testing applications that incorporate Kafka. Find out more in our deep dive","description":"Conduktor Testing provides a simple, visual interface for testing applications that incorporate Apache Kafka. Find out more in our deep dive","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"29afTe3OdmUn0bIlX1bPR8","type":"Asset","createdAt":"2022-09-23T08:20:05.110Z","updatedAt":"2022-09-25T13:57:08.465Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"testing-blog","description":"Conduktor Testing ","file":{"url":"//images.ctfassets.net/lm7q36eaz111/29afTe3OdmUn0bIlX1bPR8/7a2a6b553543b481dba458680a3fa81f/blog_-_testing.png","details":{"size":51581,"image":{"width":1200,"height":640}},"fileName":"blog - testing.png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/introducing-the-conduktor-platform/"},"content":[{"nodeType":"text","value":"With the launch of the Conduktor Platform","marks":[],"data":{}}]},{"nodeType":"text","value":", we will be releasing regular deep dives into each of the new solutions offered, giving you a full understanding of what our solutions can do and how they can bring benefits. First up is a look at ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/testing/"},"content":[{"nodeType":"text","value":"Conduktor Testing","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Some of you may already be familiar with Conduktor Testing, as it has been in a small public beta for several months. If not, the premise is simple: Conduktor Testing provides a simple, visual interface for testing applications that incorporate Apache Kafka. It also enables you to easily integrate these tests into your CI/CD environment.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Why Do You Need Testing?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The bigger question of why you should test your software has been very well covered and we won't need to go over it here. The question is perhaps better phrased as to why you need a testing solution for Apache Kafka specifically. After all, if testing is so widely understood, surely there are sufficient solutions for Apache Kafka?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"However, Kafka can be a merciless master. It is not simple to design effective tests without already having a lot of Kafka knowledge. And that is fine for Kafka experts, but with its relative youth and high complexity, experts tend to be few and far between. What is necessary is not merely another way to design and build tests in Apache Kafka, but a method that can be both fast & accessible to QAs, Business analysts, and less senior developers also.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Perhaps you could get away with a substandard solution or even ignore testing entirely, but Kafka is increasingly used to support vital infrastructure in organizations. In a microservices world, even small changes can ripple out to cause big impacts, and you simply cannot afford to miss problems too often.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What Conduktor Testing Can Bring","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As a visual test editor, you can quite literally see how simple it is to design a test and understand the flow of business data. Let's use an example and compare and contrast with a test via Testcontainers, a commonly used Java library. In this example, we will not be doing anything too complicated. We merely create a test that consists of a consumer & a producer, and involves checking business data. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Below is an example test, taken from the Testcontainers ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://github.com/testcontainers/testcontainers-java/blob/master/modules/kafka/src/test/java/org/testcontainers/containers/KafkaContainerTest.java"},"content":[{"nodeType":"text","value":"repository","marks":[],"data":{}}]},{"nodeType":"text","value":":","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1ASgYLpXB4yiMpzxGaiXtN","type":"Entry","createdAt":"2023-01-18T17:36:14.946Z","updatedAt":"2023-01-18T17:36:14.946Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"java","code":"protected void testKafkaFunctionality(\n String bootstrapServers, int partitions, int rf) throws Exception {\n try (AdminClient adminClient = AdminClient.create(ImmutableMap.of(\n AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers));\n KafkaProducer producer = new KafkaProducer<>(\n ImmutableMap.of(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,\n bootstrapServers, ProducerConfig.CLIENT_ID_CONFIG,\n UUID.randomUUID().toString()),\n new StringSerializer(), new StringSerializer());\n KafkaConsumer consumer = new KafkaConsumer<>(\n ImmutableMap.of(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,\n bootstrapServers, ConsumerConfig.GROUP_ID_CONFIG,\n \"tc-\" + UUID.randomUUID(),\n ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, \"earliest\"),\n new StringDeserializer(), new StringDeserializer());) {\n String topicName = \"messages-\" + UUID.randomUUID();\n Collection topics = Collections.singletonList(\n new NewTopic(topicName, partitions, (short) rf));\n adminClient.createTopics(topics).all().get(30, TimeUnit.SECONDS);\n consumer.subscribe(Collections.singletonList(topicName));\n producer.send(new ProducerRecord<>(topicName, \"testcontainers\", \"rulezzz\"))\n .get();\n Unreliables.retryUntilTrue(10, TimeUnit.SECONDS, () -> {\n ConsumerRecords records =\n consumer.poll(Duration.ofMillis(100));\n if (records.isEmpty()) {\n return false;\n }\n assertThat(records)\n .hasSize(1)\n .extracting(\n ConsumerRecord::topic, ConsumerRecord::key, ConsumerRecord::value)\n .containsExactly(tuple(topicName, \"testcontainers\", \"rulezzz\"));\n return true;\n });\n consumer.unsubscribe();\n }\n}"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"\nThis is not a massive amount of code by any means, but it's complicated and not something that anyone could understand without prior Kafka knowledge.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Now let's see the same thing created with Conduktor Testing:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7LSsAzMGkpsyC5ts9ljcXP","type":"Asset","createdAt":"2022-09-23T08:58:31.343Z","updatedAt":"2022-09-28T11:18:16.489Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Testing-recording-blog","file":{"url":"//videos.ctfassets.net/lm7q36eaz111/7LSsAzMGkpsyC5ts9ljcXP/8251908957071dbde45e5a15255e2793/Recording_2022-09-21_at_16.13.44.mp4","details":{"size":30772133},"fileName":"Recording 2022-09-21 at 16.13.44.mp4","contentType":"video/quicktime"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Pretty seamless, right? That ease of use doesn't just make the lives of Kafka experts easier, but it also extends testing and the understanding of tests to the wider team. Product managers, data analysts, and QA engineers can all create business tests and acknowledge the results.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Every step of the test is visible. This is particularly important for pinpointing failures, as it shows you where in an end-to-end test that a problem actually occurred. Arming team members with this knowledge make tests far more actionable. You'll know where in a streaming pipeline something has failed, and spend less time in a war room trying to decipher the issue. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"But it isn't just about having a simple way to make a test. The bigger advantage comes once you're expanding testing to cover more and more of your Kafka applications. Chances are, a lot of the tasks that these applications perform will be similar, with similar testing needs. Deploying more tests with Conduktor is easy, because your test scenarios are saved and reproducible. And since test scenarios are built step-by-step, it is easy to edit for the specific needs of each situation, saving hours of work for every additional test you need to perform. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor's automation capabilities can lighten the load even further. Conduktor Testing supports automated execution of Test Scenarios via the CI Agent. This process enables you to run collections of test scenarios from within your build pipeline. With this feature you will be able to capture regressions when making changes to your applications, minimizing the risk of introducing failures.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Other Key Features","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Moving on from the core visual test editor, let's take a look at some of the other features you might want to know about","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Test Suites and Scenarios","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Tests themselves are organized in two ways. Each individual test is termed a \"Test Scenario\". ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Scenarios can visually represent all the different tasks that go into a single test, and can be reproduced and edited endlessly to suit the needs of different users.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A collection of Test Scenarios is called a Test Suite, enabling you to create logical groups of different scenarios. ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For example, you may group all tests relating to a specific feature or domain in one suite.","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Test Environments","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You can create environments for different tests that enable you to customize certain parameters and configurations. ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Environment variables can be used in your test scenarios allowing you to rapidly alter your test parameters. ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You could create a cluster configuration variable, enabling you to execute the same test across development, staging, and production clusters.","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Workspaces","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Workspaces are designed for collaboration. A workspace is a collection of Kafka clusters, environments, and associated tests. Workspaces make it easy to organize your personal work, or divide tests by teams, or other categories.","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Domain-Specific Language","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Conduktor Domain-Specific Language (DSL) is a specialized language enabling rapid test declaration and version control of your test scenarios. ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When authoring test scenarios, you can switch between the standard visual interface and a code representation.","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Metric Insights","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Whenever you execute a test, Conduktor tracks certain metrics for you to enable you to continually optimize what and how you test. ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"These are illustrated for you under the Insights tab within the Testing interface, giving you graphical representations of statistics like your total pass/fail rate.","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"HTTP Support","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"It's common practice for systems to integrate both Kafka and HTTP. Picture this, a message is posted to a REST API and is subsequently triaged into a Kafka topic.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are multiple stages here at which we must test the flow of business data. To test the sequence of events in its entirety, you need a consolidated interface that supports both protocols. ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Postman won't be enough here, but we can definitely help! Testing provides the possibility of combining your Kafka tests alongside HTTP(REST API) testing, providing a unique methodology for testing business scenarios.","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Random Data Generation","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To get the most out of testing you need to be able to throw all kinds of different records at your system. Conduktor Testing can do this for you, automatically generating different data for your scenarios.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Create entire, randomized messages, or replace individual keys/values within JSON messages.","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Random, Contextual Data Generation ","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"What is more, we even support contextual randomness. This allows you to specify fields, for example, you could generate randomIP, randomCountryCode, or randomUUID. ","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Loading CSV Data","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If you prepare real or mock data for your tests, then you won't need to rely on our random generators. Instead, you will be able to load from external files.","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Schema Registry Integration ","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You can integrate Schema Registry into your testing scenarios, with support for ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Avro","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":", ","marks":[],"data":{}},{"nodeType":"text","value":"Protobuf","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":", and ","marks":[],"data":{}},{"nodeType":"text","value":"JSON","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" schema registry types.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Filter messages and create checks based on the schema IDs.","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Integrable in any CI/CD Environment","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As mentioned above, all of your test scenarios can be automated through your chosen CI/CD tool. ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Continuous testing supports regression testing and ensures that you do not introduce errors in your systems. ","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data Redaction Controls","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Restrict the level of data that's stored pertaining to your tests. For example, Kafka message data, or data associated with checks. ","marks":[],"data":{}}]}]}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Formats, Serializers & Deserializers","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Support for all primitive data types and serializer/deserializer formats (Avro, Protobuf, JSON, MessagePack)","marks":[],"data":{}}]}]}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"How Do I Get Started? ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor Testing is available to users via the Conduktor Platform. It's accessible through Docker or as a web app. ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Get started today ","marks":[],"data":{}}]},{"nodeType":"text","value":"and execute up to 500 tests/month at no cost at all. Testing works with all major Kafka flavors including local clusters.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Summary","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Testing was one of the most demanded features from Conduktor users, for good reason as demonstrated in this post. We have been very carefully ","marks":[],"data":{}},{"nodeType":"text","value":"testing ","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"the new product over the past few months with a small, public beta, but we're delighted to finally make it available for everyone with the launch of the Conduktor Platform. ","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1JmKff2d8J5C93f4yUSfCV","type":"Entry","createdAt":"2021-12-08T13:19:31.889Z","updatedAt":"2021-12-08T13:19:31.889Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"James","title":"Technical Product Manager","linkedIn":"https://www.linkedin.com/in/james-white-301b5550/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"65GcHlPUFtafUImTScSJjv","type":"Asset","createdAt":"2021-12-08T13:18:31.985Z","updatedAt":"2022-09-25T14:00:19.941Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"james","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/65GcHlPUFtafUImTScSJjv/d3f2ce92a6a6e7c8b4abc37b3da6fafa/james.png","details":{"size":47354,"image":{"width":192,"height":192}},"fileName":"james.png","contentType":"image/png"}}}}},"publishDate":"2022-09-28T12:00+01:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5fNk4IQgmoX2QmWWEedirp","type":"Entry","createdAt":"2022-09-15T16:44:40.973Z","updatedAt":"2022-09-25T13:32:15.908Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Building Real Time Applications Could Be Easier and Faster with Kafka Streams","description":"Kafka Streams is a simple library built on top of Kafka that is used to build real-time streaming applications","noIndex":false}},"slug":"building-real-time-applications-with-kafka-streams","title":"Building Real Time Applications with Kafka Streams","navbarDescription":"Kafka Streams is a library built on top of Kafka that enables you to build real-time streaming applications.","description":"Kafka Streams is a simple library built on top of Kafka that enables you to build real-time applications. In this article, we'll look at how Kafka Streams can be used.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"464SWdrqd5kb2cLr6eP35E","type":"Asset","createdAt":"2022-09-15T16:44:37.827Z","updatedAt":"2022-09-25T16:55:52.253Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":5,"locale":"en-US"},"fields":{"title":"Kafka Streams","description":"Kafka Streams Cover Image","file":{"url":"//images.ctfassets.net/lm7q36eaz111/464SWdrqd5kb2cLr6eP35E/479272d5a11dbe62e5952953cccff510/blog_building_real_time_applications.webp","details":{"size":49354,"image":{"width":1200,"height":630}},"fileName":"blog_building_real_time_applications.webp","contentType":"image/webp"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When we think of real-time streaming applications we usually think of data processing, map-reduce, hive, sparks, and data analytics. While this is true for streaming applications, there is another set of streaming applications where data is consumed from upstream, processed through some business logic and then sent downstream. Kafka Streams is a simple library built on top of Kafka that is used for the second type of streaming applications or microservices, which are more focused on the core application domain and less on analytics. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In Kafka Streams, a \"Stream\" represents the unbounded stream of data, which is consuming data from a Kafka topic. It might be working with one or more partitions based on parallelism built in an application.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka Streams is developed on top of Kafka to hide some of the basic details about creating producers, consumers, managing transactions, and error handling; hence developers can focus on just writing code related to business logic. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Any JVM application which uses the Kafka streams library is considered a Stream application working with real-time data.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"While using Kafka streams, the upstream needs to be a Kafka topic and the downstream could be a Kafka topic, API call, or database.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let’s look at a few important keywords to help understand Kafka streams:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Source topics:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Topics from which data is consumed. It's also called upstream for Kafka Streams applications.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Sink","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": Downstream where data is published/stored after processing. It could be another Kafka topic, database, or calls to external services.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Topology","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": DAG(Directed Acyclic Graph) which contains information about source topics, tasks, and sink topics. Nodes represent stream processors/operations and edges represent the stream of data.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Stream processor/tasks","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": Operations to be performed on data consumed from source topics. A few common operations to be performed on data could be: Filter, Map, Join, and Aggregate and they all become part of a single topology.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"KStream","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": Represents an abstraction on records flowing through the topics. Compared to a database table, every new record coming is an INSERT operation.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"KTables","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": Represents an abstraction on a changelog stream. Compared to a database table, every new record coming is an UPSERT operation. If a key already exists the record is updated, otherwise one is inserted.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Internal topics","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": KStream supports stateful operations; to store the intermediate state of these operations it creates internal topics also known as stream topics.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Local state store","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": Intermediate states are stored in a local DB(by default rockDB) for faster access which is maintained by KStream itself. States between internal state and the local DB are synced at certain time intervals. ","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Using Kafka streams library an application defines topology by specifying source topics, operations to be performed on records, and the sink to publish/store data.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The bootstrapping details about creating Kafka consumers and producers are hidden and an application is focused on implementing the core business domain. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A basic topology will look like the following:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2ah78bjW1RWvocb3K94Dcu","type":"Asset","createdAt":"2022-09-15T16:46:34.778Z","updatedAt":"2022-09-25T18:40:36.466Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Streams Topology","description":"A Kafka Streams Topology","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2ah78bjW1RWvocb3K94Dcu/017e78b33224e3f2be254eb722ec88af/Frame_38.png","details":{"size":200418,"image":{"width":800,"height":1181}},"fileName":"Frame_38.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"\nAnd the corresponding code for it will be: ","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4J1QrDdyGNEvUip8DbTMMv","type":"Entry","createdAt":"2023-01-18T17:39:43.389Z","updatedAt":"2023-01-18T17:39:43.389Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"java","code":"public Topology getTopology(StreamsBuilder kStreamBuilder) {\n final KStream < String, User > userList = kStreamBuilder.stream(\"source-topic\");\n userList.map((k, v) -> KeyValue.pair(k, new User(v.getName(),\n v.getFavoriteNumber() + 1, v.getFavoriteColor()))).to(\"sink-topic\", Produced.with(Serdes.String(), new AvroSerde()));\n System.out.println(\" USers from topics -------\");\n userList.print(Printed.toSysOut());\n return kStreamBuilder.build();\n}"}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Advantages of using Kafka Streams in building real-time applications","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Having a stream and database together:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" When implementing any stream applications, a database is also needed. So databases and streams go hand in hand. Kafka streams provides this functionality through KStream and KTable. In Kafka streams, a database can be seen as both a stream and a table which enables it to perform operations like aggregations. KTable represents a changelog topic where each record is an update on the previous record, hence why it behaves like a database table.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Stateful and stateless processing:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" One of the examples of a stateful operation is aggregation, like aggregating the number of clicks for a user where an application might need to maintain count. Kafka streams provides a simple Aggregates API to achieve a lot of stateful operations. Local state store, changelog topics and KTables make it easy to achieve stateful processing even if it's across transactions. Examples of stateless operations and transformations include filter, which is performed on every record and which can also be achieved through APIs provided by Kafka streams.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Windowed operations:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Kafka streams provides APIs for the use cases required to perform stateful and stateless operations for incoming records within certain durations, for example, the number of orders processed in the last minute or performing the join operation for records coming in 2 topics for the last 30 seconds. Kafka streams support for windowed operations avoids the need to have a separate DB to maintain states or intermediate results.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Fault tolerance: ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"Fault tolerance of the system can be achieved by specifying custom error handlers through properties and Kafka streams takes care of handling recoverable and non-recoverable exceptions internally without writing a single line of code. Kafka streams use internal topics and local state stores to save intermediate states. In case of error, instead of starting the entire process from the beginning, it can be restarted from the last saved intermediate state, hence improving performance in error scenarios.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Transactions support","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": Kafka Streams supports both at least once or exactly once semantics for transactions, which can be configured by simple properties. In case of errors, retrying entire transactions from source to sink can be achieved by just specifying the property for prcocessing.guarantee. ","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Considerations","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"It's easy to use Kafka streams and it provides so many features out of the box which is definitely not easy while using plain Kafka. When deciding to use Kafka streams, below are a couple of factors that need to be considered:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Increased configuration complexity:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" One of the complexities with Kafka is having too many configurations and relationships between those configurations, preventing you from achieving the best performance for a given load. When working with Kafka streams along with Kafka configurations, the configurations and relationships between them must be considered.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Difficulty in debugging:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Kafka Streams becomes a black box on top of Kafka, and in case of errors it does not provide much in the way of logs to understand what should be going behind the scenes. Entire topologies get broken into multiple tasks during execution making it more difficult to understand the origin of a problem.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Internal topics:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Kafka streams maintains internal topics to support KTables, windowed operations, etc. These internal topics might start consuming a considerable amount of space and cause multiple I/Os between Kafka and applications, potentially degrading performance. Internal topics and their usage needs to be understood very well in order to optimize performance for space- and time-critical applications. Unnecessary usage of KTables will create more internal topics which might cause performance issues in an application.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Local state store size: ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"Similar to internal topics, implementation and usage of local state stores is a little complex. If one application instance starts processing a large amount of data, the local state store might grow considerably, and this extra space needs to be considered while calibrating infrastructure for applications.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Stateful operations behavior:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" With Kafka streams, performing stateful operations is easier but sometimes it might cause performance issues or errors in a system. If operations start causing transactions to timeout or message size becomes too large to handle, then problems will occur.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Complexity of the built-in topology:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" As one topology could be a combination of many sub-topologies, just by looking at the code it's very difficult to understand how execution will be carried out, how internal topics will be used and how local state store will be used. You might have to depend on external tools which can provide a better pictorial presentation of topologies to understand the usage of internal state stores and topics.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data duplicity:","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" Though Kafka streams supports exactly-once semantics, in one specific scenario while performing a join between two KTables it will end up generating duplicate records. This is documented as a known issue (","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://issues.apache.org/jira/browse/KAFKA-4609#:~:text=When%20caching%20is%20enabled%2C%20KTable,results%20for%20the%20same%20key"},"content":[{"nodeType":"text","value":"KIP","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":") with no solution yet. ","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Having worked with Kafka streams for over a year, I think it's a good library to be considered if solutions involve Kafka but consider having small topologies where transactions can be short and simple, can be understood easily, and do not cause timeouts. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"\n\n\n","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"oH08Uc4XnkIX8LuvAXRpF","type":"Entry","createdAt":"2021-11-05T10:31:13.642Z","updatedAt":"2022-09-25T11:45:59.527Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Roch","title":"Software Engineer","github":"https://github.com/rokh-conduktor","linkedIn":"https://www.linkedin.com/in/rdardie/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"38BbVz2gNg5sHDusF2emge","type":"Asset","createdAt":"2021-11-05T10:31:09.458Z","updatedAt":"2022-09-25T14:02:06.804Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Roch","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/38BbVz2gNg5sHDusF2emge/4548b1829836d06e47702229f10bf7d2/roch.png","details":{"size":53264,"image":{"width":192,"height":192}},"fileName":"roch.png","contentType":"image/png"}}}}},"publishDate":"2022-09-01T15:00+01:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5uIG1aIgmJoEoP6Ti5anXk","type":"Entry","createdAt":"2022-07-15T09:14:13.868Z","updatedAt":"2022-09-25T13:39:44.084Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Distributed Systems","description":"A distributed system is a computing system in which the various components or nodes are spread out across a network of computers","noIndex":false}},"slug":"distributed-systems","title":"Distributed Systems","navbarDescription":"In this article, we will look at what a distributed system is, how they work, and the positives and negatives of such systems.","description":"A distributed system is a computing system in which the various components are spread out across a network of devices. The majority of modern computing involves distributed systems in some way and we'll look in more depth below. ","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2qti7gIzDnj93j4my3b83W","type":"Asset","createdAt":"2022-07-15T09:29:24.624Z","updatedAt":"2022-09-25T16:56:14.632Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":5,"locale":"en-US"},"fields":{"title":"distributed-systems","description":"Distributed Systems","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2qti7gIzDnj93j4my3b83W/402eb98aa44a9190b4066052868c4a88/blog_distributed_systems.webp","details":{"size":59942,"image":{"width":1200,"height":630}},"fileName":"blog_distributed_systems.webp","contentType":"image/webp"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Nearly everything a modern business does will involve software in some way, but the days of isolated mainframes running applications are long gone. Products and applications are too complex, with too many requirements. The majority now run on distributed systems that spread the burden. In this article, we will look at what a distributed system is, how they work, and the positives and negatives of such systems.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is a Distributed System?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As the name implies, a distributed system is a computing system in which the various components or nodes are spread out across a network of computers (or virtual machines, containers, or any node that can connect and handle basic tasks). Though physically separated, the nodes are linked together and pool their resources to maximize efficiency when running a program.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Practically every form of computer network can function as a distributed system in some sense, but early forms of distributed systems were difficult to setup and maintain. In the modern environment, distributed systems will be cloud-based, operating over the internet. Most SaaS applications will function as distributed systems, with end users having access to a managing application that appears to be a single interface operating in one place, but will rely on cloud computing power to process tasks. When you use a ridesharing app like Uber or Lyft, your only interaction with the system will be an app on your phone, but there will be hundreds if not thousands of components elsewhere that work together to deliver rides.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Parts of a Distributed System","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Distributed System Architectures","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The most common form of a distributed system is the client-server model, in which multiple clients make requests of a central server, which apportions resources amongst clients. Peer to Peer systems are also common. In Peer to Peer, each node is an equal participant, with work divided up equally and each node capable of acting as either a client or a server.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In addition to the two well-known architectures above, there also exist ","marks":[],"data":{}},{"nodeType":"text","value":"three-tier","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" distributed systems and ","marks":[],"data":{}},{"nodeType":"text","value":"n-tier or multi-tier ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"distributed systems. ","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Key Features of Distributed Systems","marks":[],"data":{}}]},{"nodeType":"ordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Concurrency: In a distributed system, nodes need to be able to process tasks at the same time, without issues related to sequence or reference to a global clock.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Scalability: The purpose of distributed systems is to be able to handle more complex applications, which means having more power. This power is achieved by scaling the size of the network, adding more components when needed.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Fault tolerance: Distributed systems need to be robust to individual node failures, continuing to function even in the event of a localized fault. ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Openness: The software running on a distributed system needs to be open to sharing and capable of distributing workload according to available resources","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Transparency: Each individual node should appear to be a singular, capable device. Meanwhile, the nodes themselves need to have a certain level of access to other nodes in the system for communication.","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Types of Distributed System","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Distributed Systems appear almost everywhere and as such there are thousands of possible examples all around. Practically every internet-enabled application is today built as a distributed system.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The earliest examples were the simple computer networks that originated in the early computing era around the 70s and 80s. Local area networks did function as distributed systems, and many of the technologies developed for LAN systems eventually led to the development of the internet on which most distributed systems are built today.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Some of the common internet-powered distributed systems you may see include cryptocurrency processing, distributed compute systems that power scientific research, peer-to-peer file sharing, online video games, distributed databases, and real time distributed systems such as logistics tracking, ride-hailing applications, and flight control systems.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Away from the internet, most telecommunications networks are also distributed systems. Arguably, early telephone networks were the original distributed systems, preceding LANs by several decades, with mobile networks arriving at a similar time.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Benefits of distributed systems","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are a number of reasons to build applications that utilize a distributed system. They include:","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Flexibility - Distributed systems are easy to scale, with additional nodes being added as and when they are needed.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Fault Tolerance - A single point of failure is no longer an issue when you can run multiple instances of your application across multiple data centers and cloud providers.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Low Latency - A distributed system reduces the distance between users, servers and storage systems, which helps bring down latency from milliseconds to microseconds or even nanoseconds.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Disadvantages of Distributed Systems","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Distributed systems do have their drawbacks, as explained below:","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data Consistency - In a distributed system, it is difficult to maintain data consistency. Different nodes may have multiple copies of the same data which may not be in sync with each other. This can cause problems when trying to retrieve data from one node and then using that data on another node.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Network Failure - If your network fails, then you will lose connectivity between your nodes. This can cause serious issues for your application because there could be no way for clients to access the services provided by their servers. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Security - Since distributed systems introduce many new points of access, there are also many additional vulnerabilities. Keeping the system and any associated data secure is more challenging, with modern systems needing to safely provide the right to multiple users across locations and devices.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Scheduling - Tasks being run on a distributed system need to be scheduled in a certain order, at certain times and in certain locations. While there are mature scheduling solutions, all of them still have some inefficiencies which lead to wastage of resources.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Monitoring Overhead - Keeping track of data across a distributed system is difficult, requiring a lot of resources. It’s not just a matter of making sure that application data is being used correctly either, as monitoring will also need to make sure of things like hardware usage and load balancing.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Enterprise Use Cases for Distributed Systems","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Discussing enterprise use of distributed systems is a fairly easy topic, as practically every major company will use distributed systems in some form. Much like Apache Kafka, the solution is incredibly useful and ubiquitous. Nonetheless, we will endeavor to list some of the prominent examples and possibilities below:","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Enterprise Examples","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Uber","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Uber has utilized a microservice architecture to build applications for different purposes. These microservices are deployed across Uber’s vast distributed computing power, enabling the company to deal with thousands of requests and scale rapidly across new markets. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Amazon","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"“The moment we added our second server, distributed systems became the way of life at Amazon”","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Amazon have made use of distributed computing from the very beginning of their company. ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://aws.amazon.com/builders-library/challenges-with-distributed-systems/"},"content":[{"nodeType":"text","value":"In an extensive blog","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", Jacob Gabrielson describes how distributed computing evolved at Amazon over the years. Amazon identify three different types of distributed system in use: Offline, Soft real-time, and real-time.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Enterprise Use Cases","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Looking more broadly, here are some potential use cases:","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Parallel Processing","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In the past, parallel computing and distributed systems were two separate things. Nowadays, thanks to modern operating systems, processors, and cloud services, distributed systems are widely used to enable parallel processing applications, improving speeds and quality..","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Distributed artificial intelligence","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Modern machine learning algorithms require vast sets of training data to learn from, as well as massive amounts of processing power. Distributed AI makes use of the potential of distributed systems to train from very large data sets using multi-agents more quickly.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Distributed Database Systems","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Distributed databases are databases that are located across multiple servers, possibly across multiple physical locations. Distributed databases can be either homogeneous or heterogeneous. A homogeneous distributed database will share the same DBMS and data model across every node in the network. A heterogeneous distributed database can have multiple data models and DBMSs.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka can be deployed as a platform to help enable a distributed database, and can even function as a database itself if needed. ","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Cloud Computing vs Distributed Systems","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Cloud computing is a popular buzzword in business circles, and it may seem that cloud computing is just another type of distributed system. However, the two are technically different. With a distributed system, resources are pooled from all the different nodes and tasks shared amongst the pool; the aim of such a system is to provide more power, supporting more tasks, and with greater compute. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Cloud computing simply implies a network-hosted server being used for any task. It is very easy to use cloud computing to build a distributed system, since you just need to add more servers to whatever application is running or task is being performed. But that doesn’t imply that every use of cloud computing must be a distributed system. The purpose of cloud computing is to offer on-demand environments, not the power or scalability of a distributed system.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Without distributed systems, the modern world of computing simply wouldn’t be possible. Telecommunications and the internet simply could not function without them. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As the business world continues to develop and embrace more technology, more compute, more cloud computing, and so on; so distributed systems will continue to rise in popularity. There simply isn’t a path for the modern enterprise that doesn’t involve some kind of distributed network, data store or compute. After all, this is why Apache Kafka has become so widely used, as the platform is essential for the management of data streams across distributed systems.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Given the complexity that is already present in distributed systems, businesses need any advantage they can get when it comes to building and maintaining their systems. Apache Kafka is a difficult and complex platform that can have a steep learning curve, which is why we developed Conduktor UI to take the pain out of Apache Kafka. ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"Start a free trial of the system right now to see it in action","marks":[],"data":{}}]},{"nodeType":"text","value":" and learn how easy it can be to get a Kafka deployment up and running.","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1JmKff2d8J5C93f4yUSfCV","type":"Entry","createdAt":"2021-12-08T13:19:31.889Z","updatedAt":"2021-12-08T13:19:31.889Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"James","title":"Technical Product Manager","linkedIn":"https://www.linkedin.com/in/james-white-301b5550/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"65GcHlPUFtafUImTScSJjv","type":"Asset","createdAt":"2021-12-08T13:18:31.985Z","updatedAt":"2022-09-25T14:00:19.941Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"james","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/65GcHlPUFtafUImTScSJjv/d3f2ce92a6a6e7c8b4abc37b3da6fafa/james.png","details":{"size":47354,"image":{"width":192,"height":192}},"fileName":"james.png","contentType":"image/png"}}}}},"publishDate":"2022-07-15T14:00+02:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6BRF9jsYrV9Q80C4FLpwiw","type":"Entry","createdAt":"2022-07-08T14:40:31.989Z","updatedAt":"2022-09-25T13:33:03.373Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Conduktor Now Offers Role-based Access Control at the Topic Level","description":"Topic-level role-based access control is now available in the latest versions of Conduktor.","noIndex":false}},"slug":"topic-level-rbac","title":"Conduktor Now Offers Topic-Level RBAC","navbarDescription":"Topic-level role-based access control is now available in the latest versions of Conduktor.","description":"We're pleased to announce that topic-level role-based access control is now available in the latest versions of Conduktor.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4bjG8M1d4f67GFLUxKo2z5","type":"Asset","createdAt":"2022-07-12T08:26:37.396Z","updatedAt":"2022-09-25T16:56:51.627Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"RBAC","description":"topic level role based access control announcement","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4bjG8M1d4f67GFLUxKo2z5/33acaea7cb014467506c722db232a8a5/blog_topic_level_rbac.webp","details":{"size":19234,"image":{"width":1200,"height":630}},"fileName":"blog_topic_level_rbac.webp","contentType":"image/webp"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For those users who have paid careful attention, you may have noticed a new feature was recently rolled out for Conduktor UI. Topic-level role-based access control (RBAC) was one of the most requested features we received; it is now available in the latest versions of Conduktor.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If you’re wondering what all the fuss is about, it is worth understanding just what RBAC is and why it can be such a powerful feature. As the name implies, RBAC allows an admin to determine what different users can do with a system by assigning roles: some users could have roles that permit reads only, for example. In Conduktor, RBAC can be used to restrict read and write access per topic, meaning you can control a user’s ability to use the consumer/producer functionality depending on the topic.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Why use RBAC?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The major benefits of this feature relate to compliance, security, and privacy. For businesses and enterprises making use of Kafka, there is always going to be a need to perform data governance. There is no reason for users to have access to every single facet of Apache Kafka in most cases. AWS has the principle of “least privilege”: the idea that only the permissions required to complete a task should be granted. It is one of the AWS Well-Architected best practices and can be a good addition to governance policies.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Looking at security, there is also the possibility of users making mistakes or deliberately doing damage during routine operations. Restricting permissions limits the damage that can be done. Potentially, these restrictions can also help against phishing attacks and other malicious actions. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Given these risks, enterprises would have to deploy manual security measures and authorizations if RBAC wasn’t available, so having the feature also makes it easier and quicker to set up. Authorization can be done at scale by administrators.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"How can I implement RBAC?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Topic level RBAC is available for Conduktor UI for enterprise users. A full guide to getting setup is available in the ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/desktop/account-management/managing-permissions-rbac/"},"content":[{"nodeType":"text","value":"Conduktor documentation","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", but a brief guide is detailed below:","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To get started with RBAC, you will need to head to your account on ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/"},"content":[{"nodeType":"text","value":"conduktor.io","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":". In the account settings, head to the “My Clusters” tab:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5MXKAfJJ4VETkyWN2yJviU","type":"Asset","createdAt":"2022-07-08T14:43:09.862Z","updatedAt":"2022-09-25T11:44:07.822Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"cdk-clusters","description":"My Clusters view","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5MXKAfJJ4VETkyWN2yJviU/bf32af6c8c83d93cd15404683d9241b3/rbac-blog-1.jpg","details":{"size":357466,"image":{"width":1600,"height":1080}},"fileName":"rbac-blog-1.jpg","contentType":"image/jpeg"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"RBAC is available at both the cluster level and the topic level. To enable it at a cluster level, you simply need to click on the “Manage access” button. This will present you with the following screen:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5anyLFJtTSFQSnJv5Ij0xE","type":"Asset","createdAt":"2022-07-08T14:43:53.495Z","updatedAt":"2022-09-25T11:44:12.386Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"cdk-default-rbac","description":"RBAC view","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5anyLFJtTSFQSnJv5Ij0xE/9835d380458dd3b84e21395103f2caa5/rbac-blog-2.jpg","details":{"size":290427,"image":{"width":1600,"height":1080}},"fileName":"rbac-blog-2.jpg","contentType":"image/jpeg"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Turn on the “Access Control” setting to enable RBAC for clusters. Below, the “Topic permissions” section is where you can setup RBAC at the topic level. Click on “Define topic permissions to do that:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"lqsfjYVXa7GVY47G1xAk4","type":"Asset","createdAt":"2022-07-08T14:48:31.155Z","updatedAt":"2022-09-25T11:44:16.891Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"topic-permissions","description":"topic Permissions view","file":{"url":"//images.ctfassets.net/lm7q36eaz111/lqsfjYVXa7GVY47G1xAk4/7427817cd45df256d9365b929e297e65/rbac-blog-3.jpg","details":{"size":366441,"image":{"width":1600,"height":1080}},"fileName":"rbac-blog-3.jpg","contentType":"image/jpeg"}}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"What permissions can be applied?","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ability to consume","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ability to produce","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ability to update topic configuration","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ability to import data","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ability to update leader election","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ability to update partitions","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ability to cleanup partitions","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ability to change replication factor","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ability to empty topic","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ability to check reassignments","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ability to update replication factor","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Ability to read topic size","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Once you’ve determined what permissions you will need, they will be applied once you connect to a cluster from within Conduktor UI.","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1JmKff2d8J5C93f4yUSfCV","type":"Entry","createdAt":"2021-12-08T13:19:31.889Z","updatedAt":"2021-12-08T13:19:31.889Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"James","title":"Technical Product Manager","linkedIn":"https://www.linkedin.com/in/james-white-301b5550/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"65GcHlPUFtafUImTScSJjv","type":"Asset","createdAt":"2021-12-08T13:18:31.985Z","updatedAt":"2022-09-25T14:00:19.941Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"james","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/65GcHlPUFtafUImTScSJjv/d3f2ce92a6a6e7c8b4abc37b3da6fafa/james.png","details":{"size":47354,"image":{"width":192,"height":192}},"fileName":"james.png","contentType":"image/png"}}}}},"publishDate":"2022-07-12T13:00+01:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2PFK6hkq1NnSMLLgOx0jsr","type":"Entry","createdAt":"2022-06-14T08:44:05.164Z","updatedAt":"2022-09-25T13:37:49.934Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Understanding the Apache Kafka Architecture","description":"Apache Kafka offers a simple yet powerful architecture that can handle a wide variety of use cases. Learn the details of Kafka with Conduktor","noIndex":false}},"slug":"kafka-architecture","title":"Understanding the Apache Kafka Architecture","navbarDescription":"Apache Kafka offers a simple yet powerful architecture that can handle a wide variety of use cases","description":"Apache Kafka offers a simple yet powerful architecture that can handle a wide variety of use cases. In this article, we will explore the details of Kafka's architecture.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1NqYBAkMF3KZk90t9eUk18","type":"Asset","createdAt":"2022-06-17T13:30:17.682Z","updatedAt":"2022-09-25T16:57:21.991Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":5,"locale":"en-US"},"fields":{"title":"Kafka Architecture","description":"Apache Kafka architecture abstract","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1NqYBAkMF3KZk90t9eUk18/058bddfd29f8ec80c2ff09ac530b4e6e/blog_understanding_apache_kafka_architecture.webp","details":{"size":158314,"image":{"width":1200,"height":630}},"fileName":"blog_understanding_apache_kafka_architecture.webp","contentType":"image/webp"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/"},"content":[{"nodeType":"text","value":"Apache Kafka","marks":[],"data":{}}]},{"nodeType":"text","value":" is an open-source ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/distributed-systems/"},"content":[{"nodeType":"text","value":"distributed system","marks":[],"data":{}}]},{"nodeType":"text","value":" consisting of servers and clients, primarily used for data streaming. Since its initial development by LinkedIn in 2011, Apache Kafka has become the standard for enterprises and other businesses looking to build data streaming pipelines.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka is now used by more than 80% of the Fortune 100 companies. Kafka provides a simple message queue interface on top of its append-only log-structured storage medium. It stores a log of events. Data is distributed to multiple nodes. Kafka is highly scalable and fault-tolerant to node loss.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka has been deployed in sizes ranging from just three nodes to thousands of nodes. It is used extensively in production workloads in companies such as Netflix, Apple, Uber, and Airbnb, in addition to LinkedIn. The creators of Kafka left LinkedIn to form their own company called Confluent to focus full-time on Kafka and its ecosystem. Apache Kafka is now an open-source project maintained by Confluent.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"The Apache Kafka Architecture","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Perhaps one of the reasons for the success of Kafka is that its architecture is actually relatively simple, while still being highly scalable and fault-tolerant. At its heart, Kafka is simply about moving data from a producer of data to a consumer of data, which is managed via a Kafka topic.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We won’t cover every aspect in detail here, as that would make this guide far too large, but you’ll find links to in-depth looks at each aspect of Kafka’s architecture. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Producers ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"are external data sources outside of Kafka, typically applications that will integrate an Apache Kafka library in order to send data via Kafka’s Producer API. Apache Kafka supports all widely used programming languages.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/kafka-producers/"},"content":[{"nodeType":"text","value":"Read in more detail about Producers here","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Consumers ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"are the applications that receive data from Kafka, by reading data from one or more Kafka topics. As with Producers, Consumer applications integrate a Kafka client library, with libraries available for almost all widely used programming languages.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/kafka-consumers/"},"content":[{"nodeType":"text","value":"Learn more about Consumers","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Producers and Consumers exist outside of Kafka. Internally, everything in Kafka is organized into a ","marks":[],"data":{}},{"nodeType":"text","value":"Kafka Cluster","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". A Kafka Cluster contains ","marks":[],"data":{}},{"nodeType":"text","value":"Topics","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":", which organize related events or messages. The data in the topics are stored in the key-value form in binary format. Kafka topics can contain any kind of message in any format, and the sequence of all these messages is called a data stream.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Topics themselves are organized into a number of ","marks":[],"data":{}},{"nodeType":"text","value":"partitions","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". A single topic may have more than one partition; it is common to see topics with 100 partitions, and you can have hundreds of thousands if you wanted.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The number of partitions of a topic is specified at the time of topic creation. Partitions are numbered starting from 0 to N-1, where N is the number of partitions. Each partition may live on different servers, also known as ","marks":[],"data":{}},{"nodeType":"text","value":"Kafka brokers","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/kafka-topics/"},"content":[{"nodeType":"text","value":"Learn more about Topics","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Data in Apache Kafka is written and read in order, with new records appended to the end of a partition by producers. Consumers read in order from each partition, though if a consumer is reading from multiple partitions then message order cannot be guaranteed. The position of a record within a partition is known as the ","marks":[],"data":{}},{"nodeType":"text","value":"offset; ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"consumers always read from lower offsets to higher offsets, and cannot read data backwards.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With these core concepts, we can illustrate a basic example of how the Kafka architecture functions:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6NC7qTY4BJJkM1V794td1i","type":"Asset","createdAt":"2022-06-14T08:41:45.627Z","updatedAt":"2022-09-25T07:12:38.734Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Kafka Core","description":"How Consumers, Producers and Topics interact in Kafka","file":{"url":"//images.ctfassets.net/lm7q36eaz111/6NC7qTY4BJJkM1V794td1i/9866b4a8526d4ecb7d4dbcda8da8ccd9/Kafka_Topics_2.webp","details":{"size":23320,"image":{"width":1920,"height":477}},"fileName":"Kafka_Topics_2.webp","contentType":"image/webp"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A traffic company wants to track its fleet of trucks. Each truck is fitted with a GPS locator that reports its position to Kafka. We can create a topic named - ","marks":[],"data":{}},{"nodeType":"text","value":"trucks_gps","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" to which the trucks publish their positions. Each truck may send a message to Kafka every 20 seconds, each message will contain the truck ID and the truck position (latitude and longitude). In this case, the trucks (or rather, a GPS application inside of each truck) are the producers). There may be different consumers of the topic. For example, an application that displays truck locations on a dashboard or another application that sends notifications if an event of interest occurs.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A traffic company wants to track its fleet of trucks. Each truck is fitted with a GPS locator that reports its position to Kafka. We can create a topic named - ","marks":[],"data":{}},{"nodeType":"text","value":"trucks_gps","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" to which the trucks publish their positions. Each truck may send a message to Kafka every 20 seconds, each message will contain the truck ID and the truck position (latitude and longitude). In this case, the trucks (or rather, a GPS application inside of each truck) are the producers). There may be different consumers of the topic. For example, an application that displays truck locations on a dashboard or another application that sends notifications if an event of interest occurs.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Kafka Brokers and Kafka Clusters","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Above we mentioned that topics are held in a Kafka cluster, while partitions can live across multiple Kafka brokers. A single Kafka server is called a ","marks":[],"data":{}},{"nodeType":"text","value":"Kafka Broker","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". That Kafka broker is a program that runs on the Java Virtual Machine (Java version 11+) and usually a server that is meant to be a Kafka broker will solely run the necessary program and nothing else. An ensemble of Kafka brokers working together is called a ","marks":[],"data":{}},{"nodeType":"text","value":"Kafka Cluster","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka brokers store data in a directory on the server disk they run on. Each topic-partition receives its own sub-directory with the associated name of the topic. To achieve high throughput and scalability on topics, Kafka topics are partitioned. If there are multiple Kafka brokers in a cluster, then partitions for a given topic will be distributed among the brokers evenly, to achieve load balancing and scalability.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Clusters can contain any number of Kafka brokers, from just one for simple tasks to thousands for major enterprise level data management.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/kafka-brokers/"},"content":[{"nodeType":"text","value":"Read more about brokers and clusters","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Apache Kafka with Zookeeper","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The management of a Kafka cluster is currently handled by Zookeeper (but it is worth noting that Zookeeper will be deprecated in future versions of Kafka and the management done internally).","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Zookeeper is used for metadata management in the Kafka world. For example:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Zookeeper keeps track of which brokers are part of the Kafka cluster","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Zookeeper is used by Kafka brokers to determine which broker is the leader of a given partition and topic and perform leader elections","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Zookeeper stores configurations for topics and permissions","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Zookeeper sends notifications to Kafka in case of changes (e.g. new topic, broker dies, broker comes up, delete topics, etc.…)","marks":[],"data":{}}]}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A Kafka cluster may have multiple Zookeeper servers, though it will require at least 3. One server will be the leader, handling all writes, while the others are followers and handle reads. Multiple Kafka brokers can be assigned to a single Zookeeper server.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/zookeeper-with-kafka/"},"content":[{"nodeType":"text","value":"Learn about Zookeeper","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Topic Replication","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In Kafka, replication means that data is written down not just to one broker, but many. Every topic has a ","marks":[],"data":{}},{"nodeType":"text","value":"replication factor ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"which determines how many times a partition is copied to other brokers. The replication factor is set at the topic level when they are created; a replication factor of 3 means that each partition in that topic will have a total of 3 copies, 1 on the original broker and 2 other copies on separate brokers on the cluster. This feature makes Kafka highly resilient since data will be preserved even in the face of broker failures. ","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Summing up the Kafka Architecture","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To recap: producers are applications that write data to Kafka topics. Topics are made up of multiple partitions that can be spread across servers in a Kafka cluster. Consumers read data from partitions in order, tracking their position using the offset.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A single Kafka server is called a Kafka broker, and a Kafka cluster consists of a number of Kafka brokers. A broker can have multiple Kafka topics which receive the data, with each topic also having multiple partitions. Brokers are currently managed by Zookeeper. The diagram below illustrates the overall Kafka architecture:","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3PYBqflzVzcyLDD8sUCkoY","type":"Asset","createdAt":"2022-06-14T08:42:47.736Z","updatedAt":"2022-09-25T07:12:51.112Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Kafka Cluster - Fundamentals","description":"The Kafka Architecture","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3PYBqflzVzcyLDD8sUCkoY/ad1273b261a0903aa857846fe4e063cf/Kafka_Cluster__-_Fundamentals.webp","details":{"size":45748,"image":{"width":1920,"height":963}},"fileName":"Kafka_Cluster__-_Fundamentals.webp","contentType":"image/webp"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Take note from the above diagram that there are some relevant aspects that we have not mentioned in this article, such as consumer groups, but interested readers can check out our ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/kafka-fundamentals/"},"content":[{"nodeType":"text","value":"Kafka fundamentals","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" series to learn more.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Apache Kafka APIs","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"5 core APIs are available for Apache Kafka:","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"1. Producer API","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Producer API, as you might expect, enables external ‘Producer’ Applications to publish data to Kafka topics within a Kafka cluster. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"2. Consumer API","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"On the opposite end, the Consumer API is what enables external consumers to read data from Kafka topics by subscribing to topics.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"3. Streams API","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Streams API allows stream processing applications to make use of streaming data stores in order to provide real-time analytics. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"4. Connect API","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Connect API is designed for connectors which integrate popular systems with Kafka. It allows us to re-use existing components to source data into Kafka and sink data out from Kafka into other data stores.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"5. Admin API","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Admin API enables users to manage and inspect topics, brokers, acls, and other Kafka objects. ","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"The Advantages of Kafka’s Architecture","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka has a relatively simple architecture. There are only a few different parts, and once Zookeeper is deprecated everything you need to deploy a Kafka solution will be contained within Kafka itself. The simplicity of Kafka contributes to some of its advantages, enabling Kafka to be adapted for a variety of use cases and scaled as appropriate.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The key reasons to use Kafka are as follows:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Scalability","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": Multiple Producers and Consumers can read and write from Kafka at high speeds. Kafka can handle thousands of messages per second, providing access at very low latency - the system has a peak throughput of 605 MB/s and a p99 latency of 5ms. Furthermore, the spread of topic partitions across multiple servers enables consumers to take advantage of multiple disks.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Fault Tolerant","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": The replication of partitions across Kafka brokers makes the system resistant to individual failures, ensuring that a node going down will not cause data to be lost and will not interrupt the service.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Highly Integrated","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": Data provided by producers goes through Kafka and does not require any other supporting infrastructure. All widely used languages are supported and there already exist many pre-built connectors for applications, making set-up simple.","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Kafka Limitations","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"While Kafka is both a powerful and flexible stream processing platform, it is not designed for every use case and does have some drawbacks.","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Proxying millions of clients","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" for mobile apps or IoT: the Kafka protocol is not made for that, but some proxies exist to bridge the gap.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Lack of Queues: ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"Kafka is made of topics, not queues, unlike some alternatives, which means applications requiring message queues are not suitable.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Partition Numbers","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": The number of partitions in a topic is very static, having to be evaluated at topic creation - in other words, you will need to guess the right number of partitions. If adding a partition afterwards one thing to keep in mind is if messages are produced with keys. When publishing a keyed message, Kafka provides a guarantee that messages with the same key are always routed to the same partition. This guarantee can be important for certain applications since messages within a partition are always delivered in order to the consumer. If the number of partitions changes, such a guarantee may no longer hold.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Database Issues: ","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":"Apache Kafka is a database, and can be deployed to augment traditional database platforms in many cases. However, it does lack some features. Long term tiered storage is not available by default. It is not a powerful query engine. Kafka is an event streaming log with no analytical capability built in and no complex query model. ","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Additional Kafka Libraries","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Anyone who researches Apache Kafka will come across the likes of Schema Registry, Kafka Connect, Kafka Streams, and ksqlDB. These are technically not part of the internal Kafka architecture, but they are important additions to any Kafka ecosystem and it is worth having a brief understanding of what they can do:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka Streams","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": Streams is a stream processing framework that enables real-time analytics and data transformation within Kafka.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka Connect","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": Connect is an integration tool for Kafka, allowing the re-use of existing components to source data into Kafka and sink data out from Kafka into other data stores.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Schema Registry","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": Officially titled the Confluent Schema Registry, this registers data schemas in Apache Kafka in order to ensure that producers and consumers will be compatible with each other while evolving.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"ksqlDB","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":": ksqlDB is another stream processing option. As you can likely guess from the name, it enables SQL operations to be performed on Kafka topics.","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"How Conduktor Works with Apache Kafka","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/"},"content":[{"nodeType":"text","value":"Conduktor","marks":[],"data":{}}]},{"nodeType":"text","value":" is another addition to Apache Kafka, transforming the awkward CLI used by Apache Kafka into an intuitive and easy graphical user interface.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4FYbT19QBDBDHYblpQ8SZ5","type":"Asset","createdAt":"2022-09-23T08:55:17.826Z","updatedAt":"2022-09-25T13:56:52.209Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Data-explore-blog","description":"Data Exploration in Conduktor Platform","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4FYbT19QBDBDHYblpQ8SZ5/63d7bc1770177d9cb9e5997ef9c51255/blog_-_data-explorer.png","details":{"size":63003,"image":{"width":1200,"height":640}},"fileName":"blog - data-explorer.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor is a complete tool for managing Apache Kafka, integrating with all of the key Kafka technologies, end-to-end. Kafka Connect support enables you to source and sink data with just a few clicks in Conduktor. The tool works with all of the major managed Kafka services, like Confluent Cloud, Amazon MSK, Aiven and more. Clusters can be started in seconds through the UI, with everything you need to take control of the ecosystem kept in one place.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/case-studies/homepoint/"},"content":[{"nodeType":"text","value":"Homepoint","marks":[],"data":{}}]},{"nodeType":"text","value":", a US-based mortgage lender and wholesaler, deployed Kafka along with a Change Data Capture (CDC) system to deliver automated redisclosures (changes to loan conditions). Debezium was used to handle their CDC, with Conduktor providing oversight across the entire architecture.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Using Conduktor UI enabled the team at Homepoint to monitor their Kafka ecosystem easily and effectively, offering live feedback and providing filtering options that allowed Homepoint to focus on single loans whenever they needed. Homepoint saw 20% process efficiency increases through the automation, with the company stating that they achieved savings in the hundreds of thousands thanks to Conduktor.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If you’re looking to get started with Kafka, or simply want to simplify and enhance your Kafka experience, ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"start your free trial of Conduktor today","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka offers a simple yet powerful architecture that can handle a wide variety of use cases, operating at any scale while still providing outstanding performance and durable, reliable data handling. It has become the de facto standard for a number of important projects at major enterprises, being deployed for stream processing, activity tracking, logs aggregation, microservices, and data pipelines. It is not necessarily the best tool for every use, and there are alternative options out there; you may want to take a look at our articles on ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/comparing-apache-kafka-apache-pulsar/"},"content":[{"nodeType":"text","value":"Kafka vs Pulsar","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" or ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/comparing-apache-kafka-activemq-and-rabbitmq/"},"content":[{"nodeType":"text","value":"Kafka vs RabbitMQ vs ActiveMQ","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" to see what other systems might be suitable. But there is definitely a good reason why Kafka is used in 80% of Fortune 100 enterprises.","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2SX6zpOndgjWDVop0j59Uz","type":"Entry","createdAt":"2021-10-08T17:41:29.513Z","updatedAt":"2022-09-27T10:40:02.619Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":6,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Stéphane","surname":"Maarek","title":"Chief Evangelist","twitter":"https://twitter.com/StephaneMaarek","github":"https://github.com/simplesteph","linkedIn":"https://www.linkedin.com/in/stephanemaarek/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7y6tdCR7qU2yEWMJmQDS23","type":"Asset","createdAt":"2021-10-08T17:40:54.512Z","updatedAt":"2022-09-25T07:11:34.678Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Stephane Maarek","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7y6tdCR7qU2yEWMJmQDS23/523069a652132097fc9825d7831c0790/stephane-m.png","details":{"size":47223,"image":{"width":192,"height":192}},"fileName":"stephane-m.png","contentType":"image/png"}}}}},"publishDate":"2022-06-20T17:00+00:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5QddYa5jFnrvYJW0NA88ex","type":"Entry","createdAt":"2022-05-23T12:02:19.909Z","updatedAt":"2022-09-25T13:37:12.068Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Comparing Apache Kafka and Apache Pulsar","description":"Apache Kafka and Apache Pulsar are both open source messaging solutions that can cover many of the same use cases, but with some key differences. ","noIndex":false}},"slug":"comparing-apache-kafka-apache-pulsar","title":"Comparing Apache Kafka and Apache Pulsar","navbarDescription":"We look at the similarities and differences between Apache Kafka and Apache Pulsar","description":"Apache Kafka and Apache Pulsar are both open source messaging solutions that can cover many of the same use cases, but with some key differences. We'll examine their strengths and weaknesses below.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"DflaMcd4OoSDBdqsN8XC8","type":"Asset","createdAt":"2022-05-23T11:59:27.212Z","updatedAt":"2022-09-25T07:10:56.815Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"gTs2w7bu3Qo","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/DflaMcd4OoSDBdqsN8XC8/fb05a21ebc322059168ee09a21aba44f/jakc__1_.png","details":{"size":202712,"image":{"width":1200,"height":630}},"fileName":"jakc (1).png","contentType":"image/png"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka is the dominant platform for ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/what-is-data-streaming/"},"content":[{"nodeType":"text","value":"data streaming,","marks":[],"data":{}}]},{"nodeType":"text","value":" with the majority of large enterprises maintaining a Kafka ecosystem. However, it is not the only distributed messaging solution out there. We’ve previously compared Kafka with RabbitMQ and ApacheMQ, but in this blog we’ll be looking at how Kafka compares with Apache Pulsar. Like Kafka, Pulsar is maintained as an open-source solution by the Apache Software foundation. Though the two platforms were originally designed to solve wholly different use cases, there are now plenty of similarities between the two, and each can accomplish many of the tasks of the other.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is Apache Kafka?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"http://kafka.apache.org"},"content":[{"nodeType":"text","value":"Apache Kafka","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" is a distributed event streaming platform that can ingest events from different source systems at scale and store them in a fault-tolerant distributed system called a ","marks":[],"data":{}},{"nodeType":"text","value":"Kafka cluster","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". A Kafka cluster is a collection of ","marks":[],"data":{}},{"nodeType":"text","value":"brokers","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" who organize events into ","marks":[],"data":{}},{"nodeType":"text","value":"topics","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" and store them durably for a configurable time. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A Kafka topic is divided into several partitions. A partition holds a subset of events belonging to a topic. Incoming events are written to a partition sequentially, enabling Kafka to achieve a higher write throughput. Each partition is consumed by many consumers in parallel, with each consumer maintaining a unique view of the partition. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Message storage architecture and message consumption style make Kafka significantly different from typical message brokers. Also, Kafka doesn’t have the concept of queues. We will explore these differences further in the coming sections.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is Apache Pulsar?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://pulsar.apache.org/"},"content":[{"nodeType":"text","value":"Apache Pulsar","marks":[],"data":{}}]},{"nodeType":"text","value":" is a distributed messaging system that is also capable of handling message queueing. Pulsar is cloud-native and multi-tenant, capable of scaling up or down dynamically without downtime. The architecture of Pulsar is very similar to Kafka’s, with Pulsar also having clusters that are collections of brokers.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A typical Pulsar cluster will have three types of nodes: brokers receive and dispatch messages, BookKeepers handle persistent storage of messages, and ZooKeepers take care of coordination between nodes. Apache Kafka also uses ZooKeeper, but this feature will be replaced in newer versions of Kafka.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The reliance on both BookKeeper and ZooKeeper gives Pulsar a more complicated architecture than Kafka overall, particularly once ZooKeeper is replaced. However, Pulsar does support message queueing, unlike Kafka.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Kafka vs Pulsar","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Given their similarities, it is unsurprising that Kafka and Pulsar could be deployed for the same use cases. However, there are key differentiators that should influence the choice of platform.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Message Consumption","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The first difference we can observe between Pulsar and Kafka is related to how messages are handled. Though both systems transfer messages from ","marks":[],"data":{}},{"nodeType":"text","value":"producers","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" to ","marks":[],"data":{}},{"nodeType":"text","value":"consumers","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" via ","marks":[],"data":{}},{"nodeType":"text","value":"brokers","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":", in Kafka messages are pulled from the server by consumers. In Pulsar, consumers need to subscribe to topics, with messages pushed to subscribers. ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"588Ed174Y2bWkVICJHN77r","type":"Asset","createdAt":"2022-04-04T16:28:53.774Z","updatedAt":"2022-09-25T07:10:28.510Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":6,"locale":"en-US"},"fields":{"title":"Figure 03 - Pull vs. push message consumption.","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/588Ed174Y2bWkVICJHN77r/247d7398a4eaf367dff68aa3df0690fd/Group_69.png","details":{"size":33816,"image":{"width":1074,"height":633}},"fileName":"Group 69.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As with ActiveMQ and RabbitMQ, this enables Pulsar to support message queues where Kafka does not. When records are pushed to a topic in Kafka, you can either consume them directly or with a consumer group. In Pulsar, there are more options. Pulsar offers 4 subscription types: Exclusive, Failover, Shared, and Key_Shared:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Exclusive is the default method. Only one consumer can subscribe to an Exclusive topic.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With Failover, multiple consumers can attach to the same subscription, with one defined as the master consumer. If the master consumer disconnects, messages are passed to the next consumer in line.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In Shared, multiple consumers can subscribe to the same topic, with messages delivered in a round robin format. This means that strict message ordering is not guaranteed.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In Key-Shared, multiple consumers can again subscribe to the same topic, but messages are given keys that determine the consumer they are delivered to, allowing you to maintain message ordering.","marks":[],"data":{}}]}]}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7D6yvOshkDmJYB1hRjfiT0","type":"Asset","createdAt":"2022-05-23T11:55:33.805Z","updatedAt":"2022-09-25T07:11:23.317Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Pulsar Subscriptions","description":"The 4 Types of Apache Pulsar Subscription","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7D6yvOshkDmJYB1hRjfiT0/7401649d3ce14dcf4bf4ecc3b47bed15/PulsarSubscriptions.png","details":{"size":143174,"image":{"width":2880,"height":2740}},"fileName":"PulsarSubscriptions.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Message Retention","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Both Kafka and Pulsar can store messages indefinitely, though with slightly different methods. Kafka messages are durable by default, but have a retention period set by log.cleanup.policy. The default for this policy is a week. Pulsar uses Apache BookKeeper to handle persistent storage, which is a distributed write-ahead log system. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"BookKeeper creates nodes called Bookies that are assigned to individual brokers; each broker can have any number of bookies. However, BookKeeper manages storage independently of brokers, meaning you can increase capacity through BookKeeper itself, without needing to add more brokers.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Message Tiering","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Perhaps the crucial advantage of BookKeeper is that it enables tiered storage out-of-the-box for Pulsar. Kafka can support tiered storage but it is not available by default, meaning that Pulsar can deliver storage cost savings by moving older data to cheaper storage solutions.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Relying on an object storage service like S3 to persist older messages also prevents performance problems on live data when consuming a topic for old data.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Stream Processing","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka has some significant advantages in stream processing. Kafka Streams and ksqlDB enable developers to build scalable applications that can deliver all necessary stream processing functions, like joins, windowing, and aggregations. Kafka can further enhance its stream processing capabilities through the Kafka Streams API.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Pulsar is far more limited, relying on Pulsar Functions. Functions offers basic functionality but is not a true stream processing solution. It will often need to be supported by another system like Apache Flink or Apache Storm, adding further complexity to an already complex ecosystem. However, Pulsar can perform SQL on streaming data thanks to its integration with Trino.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Message Queueing","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As we mentioned above, one difference between Kafka and Pulsar is that Pulsar supports message queues. Pulsar allows the selective acknowledgment of messaging, meaning that multiple consumers can consume from one topic and process messages in parallel. With Kafka, you can only commit all messages ","marks":[],"data":{}},{"nodeType":"text","value":"up to a given offset","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":". It is not possible to acknowledge individual messages.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Data Connectors ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Kafka Connect framework enables Kafka to support a wide range of external data sources. Kafka Connect comes with “connectors” to different ecosystem components that enable moving a large data collection into and out of Kafka - connections are known as ‘sources’ and ‘sinks’ for data that is moved in and out respectively. Organizations like Confluent have created dozens of connectors for Kafka connect, greatly simplifying development work by allowing users to quickly integrate legacy databases into Kafka or populate BI tools with just a few commands.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Pulsar matches this capability through Pulsar IO, but ultimately still lags far behind Kafka for support. Kafka Connect offers more than 400 connectors, while Pulsar is still in the dozens. This gap is likely to close with time, and developers can still build their own connections for Pulsar, but it is a big advantage for getting started with a Kafka project.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Support","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If there is one reason why Kafka is the more mature and widely used platform, it is down to its enterprise and community support. Quite frankly, Kafka has more resources and more people behind it. Enterprises like Confluent, Amazon, Red Hat, and Aiven contribute to the development of the Kafka platform while also contributing hundreds of blog posts, tutorials, podcasts, and more. Of course, these enterprises offer managed Kafka services and cloud offerings that take away a lot of the complexity of dealing with Kafka. And users can further simplify the management of Kafka through tools like our own Conduktor Devtools, which provides an intuitive GUI that gives complete oversight of your Kafka ecosystem.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If you’re looking to learn anything about Kafka or ask questions, there are already loads of high quality resources. We provide hours of video lessons from Stephane Maarek, as well as the Kafkacademy for those who prefer written guides and tutorials. There are thousands of Kafka questions on Stack Overflow, hundreds of Kafka meetups, and dozens of ways to learn and test Kafka projects.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Unfortunately, Pulsar cannot match the Kafka community. Though there are some companies offering Pulsar services and supporting development, they do not have the size and resources of the likes of Confluent. The community is much smaller and you can’t expect the same quality in learning resources or other content. Even Pulsar’s biggest client, Tencent, makes much greater use of Kafka.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There is justifiably some excitement around Pulsar. It supports many of Kafka’s use cases while bringing important features out of the box. However, it is fair to say that Pulsar is still in the early stages of maturity. For those looking at a managed offering, there is simply no competition between Kafka and Pulsar. If you are evaluating the respective open-source offerings, there may be situations where Pulsar is more suited to your personal use case, and it is true that more features and better support will come with time. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If message queuing is required for your project, then Pulsar is a good option, particularly if ActiveMQ and RabbitMQ cannot cover all of your needs. For anything that involves streaming, Kafka is still the clear leader. Pulsar may one day catch up with Kafka’s feature set, but it simply isn’t there yet.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To get more out of Apache Kafka, ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"try Conduktor for free.","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2SX6zpOndgjWDVop0j59Uz","type":"Entry","createdAt":"2021-10-08T17:41:29.513Z","updatedAt":"2022-09-27T10:40:02.619Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":6,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Stéphane","surname":"Maarek","title":"Chief Evangelist","twitter":"https://twitter.com/StephaneMaarek","github":"https://github.com/simplesteph","linkedIn":"https://www.linkedin.com/in/stephanemaarek/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7y6tdCR7qU2yEWMJmQDS23","type":"Asset","createdAt":"2021-10-08T17:40:54.512Z","updatedAt":"2022-09-25T07:11:34.678Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Stephane Maarek","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7y6tdCR7qU2yEWMJmQDS23/523069a652132097fc9825d7831c0790/stephane-m.png","details":{"size":47223,"image":{"width":192,"height":192}},"fileName":"stephane-m.png","contentType":"image/png"}}}}},"publishDate":"2022-05-23T00:00-05:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1cM5epood5GDr4J6grYJLE","type":"Entry","createdAt":"2022-04-04T16:16:34.290Z","updatedAt":"2022-09-25T13:36:38.974Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Comparing Apache Kafka, ActiveMQ, and RabbitMQ","description":"Apache ActiveMQ, RabbitMQ, and Apache Kafka are generally considered messaging middleware. This article explores the differences between them.","noIndex":false}},"slug":"comparing-apache-kafka-activemq-and-rabbitmq","title":"Comparing Apache Kafka, ActiveMQ, and RabbitMQ","navbarDescription":"This article explores 3 messaging middlewares, comparing and contrasting the differences.","description":"Apache ActiveMQ, RabbitMQ, and Apache Kafka are generally considered messaging middleware. This article explores each technology, compares and contrasts the differences, and concludes with intended use cases.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"aCyvvwnCXoyUKaZEmVE1R","type":"Asset","createdAt":"2022-04-04T16:16:23.439Z","updatedAt":"2022-09-25T07:10:02.789Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"Unsplash Software Image","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/aCyvvwnCXoyUKaZEmVE1R/437eee7ff98ce5a324a15dc24e4f27ec/ilya-pavlov-OqtafYT5kTw-unsplash.jpeg","details":{"size":1506693,"image":{"width":3543,"height":2365}},"fileName":"ilya-pavlov-OqtafYT5kTw-unsplash.jpeg","contentType":"image/jpeg"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache ActiveMQ, RabbitMQ, and Apache Kafka are generally considered messaging middleware. They integrate heterogeneous applications and make their interactions decoupled. But, Kafka goes beyond that definition and establishes itself as an event streaming platform. Although they have overlapping features, they have been designed to serve different use cases. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Understanding these differences beforehand will save you from using the wrong messaging middleware in your next project. This article explores each technology, compares and contrasts the differences, and concludes with intended use cases.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Technology Overview","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"What is Apache ActiveMQ?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://activemq.apache.org/"},"content":[{"nodeType":"text","value":"Apache ActiveMQ","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" is an open-source, multi-protocol message broker. Written primarily in Java, it enables applications to integrate across multiple messaging protocols, including AMQP, MQTT, STOMP, and JMS.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"ActiveMQ is a message broker supporting point-to-point and publish-subscribe messaging semantics. Standard messaging constructs like queues and topics are available for various messaging use cases. ActiveMQ has a ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://camel.apache.org/components/3.15.x/activemq-component.html"},"content":[{"nodeType":"text","value":"tight integration with Apache Camel ","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"to support ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.enterpriseintegrationpatterns.com/"},"content":[{"nodeType":"text","value":"Enterprise Integration Patterns (EIP)","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" that helps integrate heterogeneous enterprise systems together. ActiveMQ leverages these patterns via Apache Camel routes deployed directly on the broker.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are currently two \"flavors\" of ActiveMQ available - the well-known \"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://activemq.apache.org/components/classic"},"content":[{"nodeType":"text","value":"classic","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"\" broker and the \"next generation\" broker code-named ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://activemq.apache.org/components/artemis"},"content":[{"nodeType":"text","value":"Artemis","marks":[{"type":"italic"},{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":". Once Artemis reaches a sufficient level of feature parity with the \"Classic\" code-base, it will become the next major version of ActiveMQ.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"What is RabbitMQ?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.rabbitmq.com/"},"content":[{"nodeType":"text","value":"RabbitMQ","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" is a message broker initially designed as a message queue or point-to-point communication model. Even though it is implemented based on ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.amqp.org/"},"content":[{"nodeType":"text","value":"AMQP","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" protocol, it now supports multiple protocols, including MQTT. RabbitMQ can also support a publisher-subscriber model and scale by distributing queues across multiple nodes. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"RabbitMQ’s primary messaging protocol, AMQP, comes with several features that make RabbitMQ a more versatile and flexible enterprise message broker. That includes broker-side message filtering and exchange bindings supporting many message routing strategies.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Written primarily in ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://rabbitmq.com/build-server.html"},"content":[{"nodeType":"text","value":"Erlang","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", RabbitMQ provides client implementations for ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://rabbitmq.com/clients.html"},"content":[{"nodeType":"text","value":"many languages","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", including Java, .NET, and Erlang.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"What is Apache Kafka?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"http://kafka.apache.org"},"content":[{"nodeType":"text","value":"Apache Kafka","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" is a distributed event streaming platform that can ingest events from different source systems at scale and store them in a fault-tolerant distributed system called a ","marks":[],"data":{}},{"nodeType":"text","value":"Kafka cluster","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". A Kafka cluster is a collection of ","marks":[],"data":{}},{"nodeType":"text","value":"brokers","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" who organize events into ","marks":[],"data":{}},{"nodeType":"text","value":"topics","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" and store them durably for a configurable time. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A Kafka topic is divided into several partitions. A partition holds a subset of events belonging to a topic. Incoming events are written to a partition sequentially, enabling Kafka to achieve a higher write throughput. Each partition is consumed by many consumers in parallel, with each consumer maintaining a unique view of the partition. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Message storage architecture and message consumption style make Kafka significantly different from typical message brokers. Also, Kafka doesn’t have the concept of queues. We will explore these differences further in the coming sections.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Apache Kafka vs RabbitMQ vs ActiveMQ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"ActiveMQ, RabbitMQ, and Kafka are generally considered message brokers, enabling asynchronous and decoupled communication among applications. Although they have some overlapping features, fundamental differences exist in their architecture, message exchange patterns, and the intended use cases.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Enterprise messaging and event streaming are different concepts that we should understand before starting the comparison.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Understanding messaging vs. streaming","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Commands and Events are two message exchanging constructs used in asynchronous communication. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A ","marks":[],"data":{}},{"nodeType":"text","value":"Command","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" represents a request to be processed. A Command has a sender and one receiver. The receiver must process the command and acknowledge the sender about the outcome, allowing the sender to expect a sense of guarantee. A loan application request is an example of a Command where the loan processor must respond to the receiver with the outcome. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Messaging is focused on exchanging Commands between senders and receivers with a strong delivery guarantee. Message queues and conversational messaging (RPC-style) are two messaging use cases.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5nTH8hgAuahU1LmxjnKTtB","type":"Asset","createdAt":"2022-04-04T16:25:11.816Z","updatedAt":"2022-09-25T07:10:10.881Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":5,"locale":"en-US"},"fields":{"title":"Figure 01 - Commands are exchanges between the sender and the receiver via a broker","description":"Figure 01 - Commands are exchanges between the sender and the receiver via a broker","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5nTH8hgAuahU1LmxjnKTtB/4b88d1f310a7d73ed756bab3738db09a/CompareKafkaFig1.png","details":{"size":93832,"image":{"width":2537,"height":639}},"fileName":"CompareKafkaFig1.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"An ","marks":[],"data":{}},{"nodeType":"text","value":"Event","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" represents a state change of an application. It is expressed in the past tense; it carries necessary information to notify receivers about the state transition (event carried state transfer). Unlike a Command, Events don’t expect a response in return and can be processed by many receivers. Events are the basis of publish-subscribe scenarios where a single event is received. ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"TeAmFfAA3bJPMoh7Yr6E7","type":"Asset","createdAt":"2022-04-04T16:25:57.299Z","updatedAt":"2022-09-25T07:10:16.774Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Figure 02 - Many subscribers receive an event sent to a topic","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/TeAmFfAA3bJPMoh7Yr6E7/09021cc916028d57a110305c75b0303e/CompareKafkaFig2.png","details":{"size":147441,"image":{"width":2497,"height":1090}},"fileName":"CompareKafkaFig2.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Streaming focuses on producing and consuming Events across different systems where delivery guarantees and acknowledgments are not top priorities. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"ActiveMQ and RabbitMQ are good examples of ","marks":[],"data":{}},{"nodeType":"text","value":"enterprise messaging","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" use cases where end-to-end delivery guarantees are expected, making them ideal for building transactional systems.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka is an ","marks":[],"data":{}},{"nodeType":"text","value":"event streaming platform","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" capable of delivering high throughput, large volumes of events to subscribers in a publish-subscribe manner.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Knowing this key difference will help you decide on the right message broker that fits into your application use case.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Message retention","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Message consumption is considered a destructive operation in message brokers such as ActiveMQ and RabbitMQ. Messages are transient by default and deleted from the broker after they are consumed and acknowledged by consumers. However, message persistence can be enabled with explicit configurations.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For example, RabbitMQ allows you to declare ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.rabbitmq.com/queues.html#durability"},"content":[{"nodeType":"text","value":"durable queues","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" for long-term message retention. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Persistent messages are written to the disk as soon as they reach the queue. Transient messages (regular messages) are memory-mapped and flushed out to the disk upon reaching memory pressure. ActiveMQ also supports two ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://activemq.apache.org/components/artemis/documentation/latest/persistence.html"},"content":[{"nodeType":"text","value":"persistence options","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"; the default file journal is a disk-based append-only structure with great performance. The JDBC Store keeps messages in a JDBC-compliant database.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conversely, messages written to Kafka are durable by default; they are not deleted after consumption. Kafka stores messages for a set amount of time and purges messages older than the retention period. That is controlled by the log.cleanup.policy, which defaults to delete where Kafka deletes messages older than the configured retention time. The default retention period is a week. When the policy is set to compact, Kafka only stores the most recent value for each key in the topic. You can find more information about Kafka message retention ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/kafka-topic-configuration-log-compaction/"},"content":[{"nodeType":"text","value":"here","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1h9kQbPXv1k0s4nrfQItsV","type":"Asset","createdAt":"2022-04-04T16:27:16.540Z","updatedAt":"2022-09-25T07:10:22.325Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Figure X - Kafka topic compaction visualized","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1h9kQbPXv1k0s4nrfQItsV/30e14545f256c2d62335b6217e08dca9/Adv_Kafka_Topic_Log_Comp_2.png","details":{"size":150993,"image":{"width":2774,"height":780}},"fileName":"Adv_Kafka_Topic_Log_Comp_2.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The takeaway here is that messaging is transient in ActiveMQ and RabbitMQ as far as the performance is concerned. You can enable message persistence at the cost of disk I/Os. Meanwhile, Kafka has been designed to retain messages in the long term.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Message consumption - push vs. pull model","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"New messages are pushed or sent to consumers from the broker in the ","marks":[],"data":{}},{"nodeType":"text","value":"push model","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":". It is the basis of the publish-subscribe pattern, where messages are pushed to all subscribers from the broker. The pull model periodically ","marks":[],"data":{}},{"nodeType":"text","value":"polls","marks":[{"type":"bold"}],"data":{}},{"nodeType":"text","value":" the broker for any new messages. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Typically, message brokers, including ActiveMQ and RabbitMQ, support both push-based and pull-based message consumption. For example, as a JMS-compliant message broker, ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://activemq.apache.org/components/artemis/documentation/1.4.0/messaging-concepts.html"},"content":[{"nodeType":"text","value":"ActiveMQ supports message queue and topic semantics","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", providing examples for the poll and push-based message consumption. RabbitMQ also supports both models with the ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.rabbitmq.com/tutorials/tutorial-two-python.html"},"content":[{"nodeType":"text","value":"work queues","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" and ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.rabbitmq.com/tutorials/tutorial-three-python.html"},"content":[{"nodeType":"text","value":"publish-subscribe","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"588Ed174Y2bWkVICJHN77r","type":"Asset","createdAt":"2022-04-04T16:28:53.774Z","updatedAt":"2022-09-25T07:10:28.510Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":6,"locale":"en-US"},"fields":{"title":"Figure 03 - Pull vs. push message consumption.","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/588Ed174Y2bWkVICJHN77r/247d7398a4eaf367dff68aa3df0690fd/Group_69.png","details":{"size":33816,"image":{"width":1074,"height":633}},"fileName":"Group 69.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"On the other hand, Kafka supports only pull-based message consumption. Messages in a Kafka partition have a monotonically increasing index called offsets. While consuming, a consumer always reads data from a lower offset to a higher offset and cannot read data backward. This implementation was made so that consumers can control the speed at which the topics are being consumed. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The consumer’s responsibility is to remember the offset of the last read message and commit it to Kafka so that it can resume reading next time.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1FtpSk8yBwbg1PltyLB584","type":"Asset","createdAt":"2022-04-04T16:30:14.715Z","updatedAt":"2022-09-25T07:10:36.035Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Figure 04 - Kafka consumers, partitions, and offsets","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/1FtpSk8yBwbg1PltyLB584/f53fe98a60ca2000c8d21102e6409c7e/Kafka_Consumers_1.png","details":{"size":61726,"image":{"width":2448,"height":736}},"fileName":"Kafka Consumers 1.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Fine-grained subscriptions and message filtering","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Message brokers like ActiveMQ and RabbitMQ offer hierarchical topics, giving consumers more subscription points and flexibility. A hierarchical topic is a structure representing a hierarchy such as /travel/flights/airline/flight-number/seat. Consumers can subscribe to different hierarchy levels with different values to select the events at a finer granularity. In addition, messaging pub/sub selectors can be used to further refine the events of interest.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For example, a consumer subscribing to ","marks":[],"data":{}},{"nodeType":"text","value":"/travel/flights/aa","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" will receive any event related to American Airlines, whereas a subscription to ","marks":[],"data":{}},{"nodeType":"text","value":"/travel/flights/aa/aa-9235","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" delivers events for a specific flight.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5x7OpNKeTfhIhVU1R7N8Ow","type":"Asset","createdAt":"2022-04-04T16:31:48.654Z","updatedAt":"2022-09-25T07:10:40.903Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Figure 05 - Hierarchical topic subscription enables broker-side filtering on messages","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5x7OpNKeTfhIhVU1R7N8Ow/e8d7996e3319dcebef82a3e16eb99e00/comparekafkafig5.png","details":{"size":77123,"image":{"width":1602,"height":1037}},"fileName":"comparekafkafig5.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"ActiveMQ supports this with ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://activemq.apache.org/wildcards"},"content":[{"nodeType":"text","value":"destination wildcards","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", a way of organizing events into hierarchies, and using wildcards for the easy subscription of the range of information interested. Apart from that, ActiveMQ supports ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://activemq.apache.org/selectors"},"content":[{"nodeType":"text","value":"JMS selectors","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", a feature allowing consumers to filter events at the broker level. RabbitMQ also leverages the features built into AMQP protocol, such as ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.rabbitmq.com/tutorials/tutorial-five-python.html"},"content":[{"nodeType":"text","value":"topic exchanges","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", bindings, and routing keys, to offer fine-grained subscriptions to topics and broker-level filtering.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conversely, Kafka doesn’t offer that level of granular subscriptions to topics. A consumer reading from a topic must do the filtering at the consumer level. ","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Message ordering","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Strict message ordering is critical for some use cases, such as financial transactions. Sometimes it can be helpful to ensure that every topic consumer sees messages arriving on the topic in the same order. However, ActiveMQ, RabbitMQ, and Kafka implement message ordering differently.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Brokers like ActiveMQ and RabbitMQ will guarantee the order of all messages sent by the same producer. However, because the broker uses multiple threads and asynchronous processing, messages from different producers could arrive to different consumers in different orders.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For example, assume two producers, P, and Q, simultaneously send messages to the same topic as follows.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7xTJpgd2uVwDSQmqLspujH","type":"Asset","createdAt":"2022-04-04T16:33:59.782Z","updatedAt":"2022-09-25T07:10:46.176Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Figure 06 - Brokers do not guarantee total message ordering","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7xTJpgd2uVwDSQmqLspujH/0bf27ddda1d62cdaebe2ed0b4199c83e/CompareKafkaFig6.png","details":{"size":100837,"image":{"width":2879,"height":1400}},"fileName":"CompareKafkaFig6.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Two different consumers could see messages arrive in the following order:","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"consumer1: P1 P2 Q1 P3 Q2 ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"consumer2: P1 Q1 Q2 P2 P3","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"ActiveMQ offers ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://activemq.apache.org/total-ordering"},"content":[{"nodeType":"text","value":"total ordering destinations","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" to preserve message ordering across all consumers for a given topic. Nevertheless, that results in a performance cost since greater synchronization is required.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"However, Kafka has a different take on total message ordering. Kafka ensures that messages with the same partition key will always end up in the same partition regardless of the producer. Messages are written in a partition based on their arrival order, guaranteeing a strict ordering for messages with the same partition key.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Messaging protocols and interfaces","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Messaging protocols provide interoperability between messaging clients and brokers, enabling clients to move across different brokers without code changes. JMS, AQMP, and MQTT are a few famous message protocols.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"ActiveMQ is a multi-protocol broker with a pluggable protocol architecture. The current version supports protocols including AMQP, MQTT, STOP, and OpenWire. Although ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://activemq.apache.org/components/artemis/documentation/latest/using-jms.html"},"content":[{"nodeType":"text","value":"JMS and Jakarta Messaging ","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":"are standardized APIs, they do not define a network protocol. The ActiveMQ Artemis JMS & Jakarta Messaging clients are implemented on top of the core protocol. In addition, ActiveMQ also provides a","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://activemq.apache.org/components/artemis/documentation/latest/rest.html"},"content":[{"nodeType":"text","value":" REST messaging interface","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"RabbitMQ also supports several messaging protocols, directly and through plugins. RabbitMQ was initially developed to support ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.rabbitmq.com/protocol.html"},"content":[{"nodeType":"text","value":"AMQP 0-9-1","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":". This protocol is the \"core\" protocol supported by the broker. Apart from that, RabbitMQ supports ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.rabbitmq.com/protocols.html"},"content":[{"nodeType":"text","value":"other protocols","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", including STOMP, MQTT, and other variants of MQTT.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conversely, Kafka has its own ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/protocol.html"},"content":[{"nodeType":"text","value":"wire protocol","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", implemented by languages including Java, Python, .NET, Node, etc. In addition, vendors like Confluent provide ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.confluent.io/platform/current/kafka-rest/index.html"},"content":[{"nodeType":"text","value":"REST proxy","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":", a RESTful interface to Kafka.","marks":[],"data":{}}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"When it comes to enterprise workloads, three technologies provide equal security, scalability, and transactions capabilities. However, things get different when we narrow down specific use cases.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"ActiveMQ and RabbitMQ have been established as enterprise message brokers, providing reliable delivery guarantees across business applications. They are configured to acknowledge each message by default and equipped with built-in recovery mechanisms like dead letter queues (DLQ) for reliability. Also, they provide interfaces to popular messaging protocols, allowing more integration points with existing enterprise systems. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Therefore, consider ActiveMQ and RabbitMQ when building enterprise applications that can’t tolerate losing a single message. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka is an event streaming platform designed to meet different expectations. Kafka has the edge over former technologies in performance, message retention, and total ordering. Kafka’s write-ahead log (WAL) employs a sequential write principal, allowing low-latency and high throughput write performances at scale. Its partition-based design enables a higher read capacity and a strict ordering of messages. Also, Kafka’s storage can be extended for the long term (with tiered storage), thus, allowing you to access real-time and historical data from a single place. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Therefore, Kafka is ideal for building applications that demand more scalability, performance, message ordering, and longer retention periods. Transactional systems and data analytics applications are popular use cases for Kafka.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Streamline your Apache Kafka with a ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"free trial of Conduktor","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2SX6zpOndgjWDVop0j59Uz","type":"Entry","createdAt":"2021-10-08T17:41:29.513Z","updatedAt":"2022-09-27T10:40:02.619Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":6,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"Stéphane","surname":"Maarek","title":"Chief Evangelist","twitter":"https://twitter.com/StephaneMaarek","github":"https://github.com/simplesteph","linkedIn":"https://www.linkedin.com/in/stephanemaarek/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"7y6tdCR7qU2yEWMJmQDS23","type":"Asset","createdAt":"2021-10-08T17:40:54.512Z","updatedAt":"2022-09-25T07:11:34.678Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Stephane Maarek","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/7y6tdCR7qU2yEWMJmQDS23/523069a652132097fc9825d7831c0790/stephane-m.png","details":{"size":47223,"image":{"width":192,"height":192}},"fileName":"stephane-m.png","contentType":"image/png"}}}}},"publishDate":"2022-04-05T00:00-05:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"50KGXpIhTYi0KTqiEMtSCf","type":"Entry","createdAt":"2022-03-24T15:36:44.895Z","updatedAt":"2022-09-25T13:33:38.753Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Testing your Apache Kafka Data with Confidence","description":"In this post, we’ll explore some methods you can use to test components within your Apache Kafka architecture. ","noIndex":false}},"slug":"testing-your-apache-kafka-data-with-confidence","title":"Testing your Apache Kafka Data with Confidence","description":"In this post, we’ll explore some methods you can use to test components within your Apache Kafka architecture. ","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"9HdCMaX1EksyRBq48MGPp","type":"Asset","createdAt":"2022-03-24T15:36:01.913Z","updatedAt":"2022-09-25T16:54:12.730Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"nordwood-themes-EZSm8xRjnX0-unsplash","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/9HdCMaX1EksyRBq48MGPp/de6d3492cca35458e623f387c384114a/The_fastest_way_to_start_kafka_ever.jpeg","details":{"size":131222,"image":{"width":1252,"height":827}},"fileName":"The_fastest_way_to_start_kafka_ever.jpeg","contentType":"image/jpeg"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"The Case for Quality","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The significance of ‘data’ and ‘events’ in modern systems cannot be refuted. As the production of data has grown exponentially, organisations have shifted towards architectures that can more effectively process large volumes of data with millisecond latency.","marks":[],"data":{}}]},{"nodeType":"blockquote","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"“In 2021, there were more than 10 billion active IoT devices” - ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://dataprot.net/statistics/iot-statistics/"},"content":[{"nodeType":"text","value":"source","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"We refer to these methods of system design as EDAs (event-driven architectures), or microservices architecture. At their core, they depend on a messaging backbone, which is often chosen to be Apache Kafka. This messaging backbone is responsible for delivering information between interlinked services. It also helps to decouple components within the system, empowering teams to own, scale and improve their ‘piece’ of the overall system with less dependencies.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"With Kafka forming a fundamental part of systems that are often responsible for real-time decision making, it’s more important than ever to establish a robust testing framework. Such a framework will ensure you maintain product quality, reduce the risk of broken contracts, and can even eliminate the possibility of a production failure. But with distributed architectures and huge volumes of data, this can be a daunting task for even the most seasoned Apache Kafka expert. In this post, we’ll explore some methods you can use to test components within your Apache Kafka architecture. ","marks":[],"data":{}}]},{"nodeType":"hr","data":{},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Manual Testing","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Suppose we want to collect geospatial data from those 10 billion IoT devices previously mentioned. The first step would be to create a Kafka topic and validate that we can produce and consume messages from it accordingly. ","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Kafka CLI","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let’s assume we’ve already started Kafka. If you’re unsure how to do this, I recommend visiting ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/starting-kafka/"},"content":[{"nodeType":"text","value":"Conduktor Kafkademy","marks":[],"data":{}}]},{"nodeType":"text","value":" for step-by-step instructions. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Using the Kafka Topics CLI, we can create a ","marks":[],"data":{}},{"nodeType":"text","value":"geo_events","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" topic with 3 partitions and a replication factor of 1 via a Kafka broker running at localhost:9092. ","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1sg9l4qXg8uY1IiPWxAxgg","type":"Entry","createdAt":"2023-01-18T17:55:22.177Z","updatedAt":"2023-01-18T17:55:22.177Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"terminalBlock"}},"locale":"en-US"},"fields":{"command":"$ kafka-topics.sh --bootstrap-server localhost:9092 --topic geo_events --create --partitions 3 --replication-factor 1"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Great, now that our Kafka topic is created we can produce a sample record as a manual test. To do this, we utilize the Kafka producer CLI:","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2IKKosURjGFlRW88OPcPAq","type":"Entry","createdAt":"2023-01-18T17:55:48.518Z","updatedAt":"2023-01-18T17:55:48.518Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"terminalBlock"}},"locale":"en-US"},"fields":{"command":"$ kafka-console-producer.sh --bootstrap-server localhost:9092 --topic geo_events\n\n> {\"timestamp\":\"1647515491\", \"deviceId\":\"1234-e4b6-5678\", \"latLong\":\"38.8951,-77.0364\"}"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Finally, let’s use the Kafka consumer CLI to read data from our topic, confirming that we can collect our geospatial messages. We will use the ","marks":[],"data":{}},{"nodeType":"text","value":"--from-beginning ","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"option to consume all historical messages, which should include the sample produced in our prior step.","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2Js7pnGCZsWDx4anYkESYQ","type":"Entry","createdAt":"2023-01-18T17:56:10.028Z","updatedAt":"2023-01-18T17:56:10.028Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"terminalBlock"}},"locale":"en-US"},"fields":{"command":"$ kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic geo_events --from-beginning\n\n> {\"timestamp\":\"1647515491\",\"deviceId\":\"1234-e4b6-5678\", \"latlong\":\"38.8951,-77.0364\"}"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Job done! To summarise, we’ve created a topic and manually tested that we can produce and consume data accordingly. That being said, the keyword here is ‘manual’. This might be sufficient for ad-hoc debugging, but not for something continuous and automated. Additionally, writing those commands was cumbersome for such basic functionality. Surely this can be achieved more easily? ","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Kafka GUIs","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Not all users that need to interact with and test Apache Kafka data are comfortable utilizing CLI tools. Perhaps I’m a data analyst that needs only to sample some messages, or a QA engineer with no prior Kafka experience. In these cases, a GUI is your friend. Equally, they can increase the productivity of the most seasoned developers when it comes to everyday Kafka tasks.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The Conduktor DevTools GUI has a rich feature set to support manual testing. Not to mention it supports starting a Kafka cluster in ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.conduktor.io/desktop/kafka-cluster-connection/starting-a-local-kafka-cluster-in-seconds/"},"content":[{"nodeType":"text","value":"seconds","marks":[],"data":{}}]},{"nodeType":"text","value":" at the click of a button. To revisit our previous example, below demonstrates how to use the producer interface to push messages into the ","marks":[],"data":{}},{"nodeType":"text","value":"geo_events","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" topic. ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"zMd5ew8H25xrCya1GBqoN","type":"Asset","createdAt":"2022-03-24T15:43:25.974Z","updatedAt":"2022-09-25T13:33:46.183Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Conduktor DevTools GUI- 'Producer' Interface","description":"Conduktor DevTools GUI - testing Apache Kafka, integration test Kafka","file":{"url":"//images.ctfassets.net/lm7q36eaz111/zMd5ew8H25xrCya1GBqoN/5e491a025c96b81ae366601f33103894/Screen_Shot_2022-03-24_at_10.42.43_AM.png","details":{"size":420301,"image":{"width":1532,"height":1230}},"fileName":"Screen Shot 2022-03-24 at 10.42.43 AM.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Not only can we forget about the Kafka CLI syntax, but we also gain new functionality such as ‘Flow’. This enables you to produce an automated stream of events with interval rules and lifecycle options. Already, you get an idea of how much more powerful our manual testing efforts can be. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The best part, we needn’t create any additional scripts. By simply navigating the options in the GUI, we can append headers to our messages, produce randomized data and configure settings such as ","marks":[],"data":{}},{"nodeType":"text","value":"acks","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" (acknowledgements). This supports testing under more technical conditions. Pretty neat, right? ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3T9ZgVLKkqMkrPn5HZpduW","type":"Asset","createdAt":"2022-03-24T15:44:45.926Z","updatedAt":"2022-09-25T13:33:51.612Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Produce to Topic: geo_events","description":"Navigate the options in the GUI, append headers to our messages, produce randomized data and configure settings such as acks in order to test Apache Kafka, integration test Kafka","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3T9ZgVLKkqMkrPn5HZpduW/e6986651decc3f8e4784c7f2c889e04e/Screen_Shot_2022-03-24_at_10.44.09_AM.png","details":{"size":317872,"image":{"width":1382,"height":1030}},"fileName":"Screen Shot 2022-03-24 at 10.44.09 AM.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As for the consumption side of manual testing, there are complementary features whereby you might otherwise have to refer to a Kafka CLI cheat sheet. For example, configuring different consumption strategies, limiting the records you consume, or filtering your messages to intercept those that match certain criteria. Most importantly, your event stream will be displayed in an easily readable table that also surfaces record metadata. ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5wqo5GRpqVScHiqJtF51Kl","type":"Asset","createdAt":"2022-03-24T15:45:33.615Z","updatedAt":"2022-09-25T13:33:55.792Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Consume from Topic: geo_events","description":"Testing Apache Kafka, integration with Apache Kafka. ","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5wqo5GRpqVScHiqJtF51Kl/39766e7d9d6f040583e88aacca6e2334/Screen_Shot_2022-03-24_at_10.45.26_AM.png","details":{"size":281152,"image":{"width":1424,"height":772}},"fileName":"Screen Shot 2022-03-24 at 10.45.26 AM.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"While Kafka GUIs help advance our manual testing efforts, there are limits to how far we can go. Most organizations are now dependent on continuous integration and continuous delivery, therefore we need a mechanism for automated testing that can form part of the SDLC (software development lifecycle).","marks":[],"data":{}}]},{"nodeType":"hr","data":{},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Integration Testing with Testcontainers","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Advancing from our manual testing efforts, ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://github.com/testcontainers/testcontainers-java"},"content":[{"nodeType":"text","value":"Testcontainers","marks":[],"data":{}}]},{"nodeType":"text","value":" is a Java library that can be used to instantiate disposable Kafka containers via Docker. They provide ports for other languages such as ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://github.com/testcontainers/testcontainers-python"},"content":[{"nodeType":"text","value":"Python","marks":[],"data":{}}]},{"nodeType":"text","value":" and ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://github.com/testcontainers/testcontainers-go"},"content":[{"nodeType":"text","value":"Go","marks":[],"data":{}}]},{"nodeType":"text","value":", and containers for other common databases such as Postgres and MySQL. ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"72YN9nuLnyMcD5XUiBzbEY","type":"Asset","createdAt":"2022-03-24T15:46:36.380Z","updatedAt":"2022-09-25T13:34:00.158Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Testcontainers","description":"A java library that can be used to instantiate disposable Kafka containers via Docker. Provides ports for languages such as Python and Go.","file":{"url":"//images.ctfassets.net/lm7q36eaz111/72YN9nuLnyMcD5XUiBzbEY/df737327f31f798fcde27e5591bcf89e/Screen_Shot_2022-03-24_at_10.46.23_AM.png","details":{"size":137846,"image":{"width":1498,"height":810}},"fileName":"Screen Shot 2022-03-24 at 10.46.23 AM.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"One of the challenges we face with integration testing is ensuring that all dependencies are available for our test to run. Essentially, we want to replicate a ‘realistic’ environment for the test execution, and we want it to be available on demand. This is particularly relevant when we want to integrate tests in our CI pipeline.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For distributed systems that are dependent on Kafka, Testcontainers helps us spin up a cluster for the lifetime of an integration test. This means our Kafka is disposed of when the execution completes. To instantiate our Kafka container we can parse a Confluent docker image. ","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6YG0b8uJKDoOtu32DniyPi","type":"Entry","createdAt":"2023-01-18T17:56:40.338Z","updatedAt":"2023-01-18T17:56:40.338Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"java","code":"DockerImageName KAFKA_TEST_IMAGE = DockerImageName.parse(\"confluentinc/cp-kafka:6.2.1\");\nKafkaContainer kafkaContainer = new KafkaContainer(KAFKA_TEST_IMAGE);"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Once our Kafka broker is up and running, we are free to execute our integration tests. Safe in the knowledge that we have a self-contained environment that’s not being shared with other teams. In a single script, it’s now possible to automate the process of creating our ","marks":[],"data":{}},{"nodeType":"text","value":"geo_events","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" topic, producing some sample records, consuming them, and asserting the data. You can find an example class in the Testcontainers ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://github.com/testcontainers/testcontainers-java/blob/master/modules/kafka/src/test/java/org/testcontainers/containers/KafkaContainerTest.java"},"content":[{"nodeType":"text","value":"repository","marks":[],"data":{}}]},{"nodeType":"text","value":". Below shows an adaptation for our use case.","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"17ih8OkZFYVhEYrQ4p4EbQ","type":"Entry","createdAt":"2023-01-18T17:57:17.383Z","updatedAt":"2023-01-18T17:57:17.383Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"java","code":"@Test\npublic void testUsage() throws Exception {\n try (KafkaContainer kafka = new KafkaContainer(KAFKA_TEST_IMAGE)) {\n kafka.start();\n testKafkaFunctionality(kafka.getBootstrapServers());\n }\n}\nprotected void testKafkaFunctionality(String bootstrapServers) throws Exception {\n testKafkaFunctionality(bootstrapServers, 1, 1);\n}\nprotected void testKafkaFunctionality(String bootstrapServers, int partitions, int rf) throws Exception {\n try (AdminClient adminClient = AdminClient.create(ImmutableMap.of(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers)); KafkaProducer < String, String > producer = new KafkaProducer < > (ImmutableMap.of(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers, ProducerConfig.CLIENT_ID_CONFIG, UUID.randomUUID().toString()), new StringSerializer(), new StringSerializer()); KafkaConsumer < String, String > consumer = new KafkaConsumer < > (ImmutableMap.of(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers, ConsumerConfig.GROUP_ID_CONFIG, \"tc-\" + UUID.randomUUID(), ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, \"earliest\"), new StringDeserializer(), new StringDeserializer());) {\n String topicName = \"geo_events\";\n String messageValue = \"{\\\"timestamp\\\":\\\"1647515491\\\",\\\"deviceId\\\":\\\"1234-e4b6-5678\\\",\\\"latLong\\\":\\\"38.8951,-77.0364\\\"}\";\n Collection < NewTopic > topics = singletonList(new NewTopic(topicName, partitions, (short) rf));\n adminClient.createTopics(topics).all().get(30, TimeUnit.SECONDS);\n consumer.subscribe(singletonList(topicName));\n producer.send(new ProducerRecord < > (topicName, null, messageValue)).get();\n Unreliables.retryUntilTrue(10, TimeUnit.SECONDS, () -> {\n ConsumerRecords < String,\n String > records = consumer.poll(Duration.ofMillis(100));\n if (records.isEmpty()) {\n return false;\n }\n assertThat(records).hasSize(1).extracting(ConsumerRecord::topic, ConsumerRecord::key, ConsumerRecord::value).containsExactly(tuple(topicName, null, messageValue));\n return true;\n });\n consumer.unsubscribe();\n }\n}"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are clear benefits of using Testcontainers for Kafka. Primarily, it provides an on-demand testing environment that we do not need to maintain or govern. Additionally, we can embed testing in the software development lifecycle. This ensures that test executions are a pre-requisite to code being deployed.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For all the positives, there are potential organizational challenges that come with utilizing Testcontainers. It’s a very developer-oriented approach, which assumes deep understanding of Kafka and its architecture. The best testing cultures encourage co-operation across product managers, data analysts, QA engineers, and developers. With no UI to support understanding, reporting, or orchestration of tests, this could reflect a technical barrier to adoption en masse.","marks":[],"data":{}}]},{"nodeType":"hr","data":{},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Kafka Streams Test Utilities","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Kafka Streams is a frequent choice for real-time stream processing. It’s a service where business logic is likely to be injected into Kafka-based applications. For example, a stateless operation like filtering a stream, or a stateful operation such as a windowed aggregation. To support testing of Streams applications, Kafka provides a test-utils artifact.","marks":[],"data":{}}]},{"nodeType":"blockquote","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The test-utils package provides a ","marks":[],"data":{}},{"nodeType":"text","value":"TopologyTestDriver","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" that can be used pipe data through a ","marks":[],"data":{}},{"nodeType":"text","value":"Topology","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" … You can use the test driver to verify that your specified processor topology computes the correct result with the manually piped in data records - ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://kafka.apache.org/documentation/streams/developer-guide/testing.html"},"content":[{"nodeType":"text","value":"Source","marks":[],"data":{}}]},{"nodeType":"text","value":"","marks":[],"data":{}}]}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To elaborate on the official Kafka documentation, the package enables us to produce messages to a mocked input topic, apply our computational logic (Topology), and check the result in a mocked output topic. The advantage here? We don’t need to depend on an embedded or external Kafka cluster. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let’s revisit our geospatial data, and see how we can test a simple Streams application. Suppose we want to filter the geo_events topic for messages containing a specific deviceId. ","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1pBWFxymELtGxkoHfUg70u","type":"Entry","createdAt":"2023-01-18T17:57:42.670Z","updatedAt":"2023-01-18T17:57:42.670Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"java","code":"public static Topology filterStream() {\n StreamsBuilder builder = new StreamsBuilder();\n KStream < String, String > stream = builder.stream(INPUT_TOPIC);\n stream.filter((k, v) -> v.contains(\"1234-e4b6-5678\")).to(OUTPUT_TOPIC);\n return builder.build();\n}"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Using the above topology to filter our stream, we can create a test case that will pipe data in from two different deviceId’s. We will then assert the outputs, validating that only the expected output, ","marks":[],"data":{}},{"nodeType":"text","value":"messageValid","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":", reaches the ","marks":[],"data":{}},{"nodeType":"text","value":"outputTopic","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":". ","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3seipbYyPui0vrJYWYRIhu","type":"Entry","createdAt":"2023-01-18T17:58:05.732Z","updatedAt":"2023-01-18T17:58:05.732Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"java","code":"@Test\npublic void shouldFilterRecords() {\n topology = App.filterStream();\n td = new TopologyTestDriver(topology, config);\n inputTopic = td.createInputTopic(App.INPUT_TOPIC, Serdes.String().serializer(), Serdes.String().serializer());\n outputTopic = td.createOutputTopic(App.OUTPUT_TOPIC, Serdes.String().deserializer(), Serdes.String().deserializer());\n String messageValid = \"{\\\"timestamp\\\":\\\"1647515491\\\",\\\"deviceId\\\":\\\"1234-e4b6-5678\\\",\\\"latLong\\\":\\\"38.8951,-77.0364\\\"}\";\n String messageInvalid = \"{\\\"timestamp\\\":\\\"1647799800\\\",\\\"deviceId\\\":\\\"9876-e1p3-6763\\\",\\\"latLong\\\":\\\"51.5072,0.1276\\\"}\";\n assertThat(outputTopic.isEmpty(), is(true));\n inputTopic.pipeInput(null, messageValid);\n assertThat(outputTopic.readValue(), equalTo(messageValid));\n assertThat(outputTopic.isEmpty(), is(true));\n inputTopic.pipeInput(null, messageInvalid);\n assertThat(outputTopic.isEmpty(), is(true));\n}"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Great, so we have demonstrated that our filter logic is behaving as expected. In the first case, our message contains the desirable deviceId and we assert that it reaches the ","marks":[],"data":{}},{"nodeType":"text","value":"outputTopic","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":". In the second case, when piping ","marks":[],"data":{}},{"nodeType":"text","value":"messageInvalid","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" into our topology, we're able to verify that ","marks":[],"data":{}},{"nodeType":"text","value":"no","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" ","marks":[],"data":{}},{"nodeType":"text","value":"message","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" is produced in ","marks":[],"data":{}},{"nodeType":"text","value":"outputTopic","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":". ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The lightweight and readable nature of test-utils make the package a great asset for unit testing Kafka Streams applications. The provided input is processed synchronously and without external system dependencies. This makes it very straightforward to confirm your Topology is producing the desired results.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"However, there is one major drawback. Because the process does not spin up a real Kafka, it makes it impossible to work with external applications to test dependencies. This limits how far our testing efforts can go with this method alone.","marks":[],"data":{}}]},{"nodeType":"hr","data":{},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Testing ksqlDB Applications","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If you're more accustomed to working with relational databases, the simplicity of using ksqlDB for stream processing will have been attractive. Much like Kafka Streams, it’s another service where business logic is applied to produce a desired output. Therefore, it’s important that we have a mechanism for testing the behavior of a ksqlDB application. There are two options in this space, and we will briefly discuss both of them.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"ksqlDB Test Runner CLI Tool","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The ksqlDB ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://docs.ksqldb.io/en/latest/how-to-guides/test-an-app/"},"content":[{"nodeType":"text","value":"test runner","marks":[],"data":{}}]},{"nodeType":"text","value":" depends on three files to validate the stream processing logic.","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"‘statements.sql’ should contain your SQL statements","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"‘input.json’ and ‘output.json’ to describe the input data and the expected output ","marks":[],"data":{}}]}]}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"54QtqnDZnnXcltctQllDJM","type":"Asset","createdAt":"2022-03-24T16:28:14.699Z","updatedAt":"2022-09-25T13:34:08.137Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"ksqlDB Test Runner CLI Tool","description":"The ksqlDB test runner depends on three files to validate the stream processing logic. Apache Kafka, testing Apache Kafka, integration with Apache Kafka.","file":{"url":"//images.ctfassets.net/lm7q36eaz111/54QtqnDZnnXcltctQllDJM/7e71c02fe07e44bca61383c1f3eff3c9/Screen_Shot_2022-03-24_at_11.27.53_AM.png","details":{"size":97465,"image":{"width":982,"height":542}},"fileName":"Screen Shot 2022-03-24 at 11.27.53 AM.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Similar to Kafka Streams test utilities, we do not require a running Kafka or ksqlDB cluster to execute these tests. The command line tool will take care of the evaluation and display a success or failure message in the terminal.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Perhaps we want to create an alias for certain IoT devices (e.g. a driver name that’s associated with the device), we could create that logic using ksqlDB:","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"WWLNaZobhbEeEpEk6U4Y5","type":"Entry","createdAt":"2023-01-18T17:59:31.713Z","updatedAt":"2023-01-18T17:59:54.884Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"sql","code":"CREATE STREAM geo_events (\n TIMESTAMP VARCHAR, DEVICEID VARCHAR, \n LATLONG VARCHAR\n) WITH (\n KAFKA_TOPIC = 'geo_events', VALUE_FORMAT = 'JSON'\n);\nCREATE STREAM geo_events_enriched AS \nSELECT \n TIMESTAMP, \n DEVICEID, \n LATLONG, \n CASE WHEN DEVICEID = '1234-e4b6-5678' THEN 'Joe Bloggs' WHEN DEVICEID = '9876-e1p3-6763' THEN 'Jane Doe' ELSE 'Unknown' END AS DRIVERNAME \nFROM \n geo_events EMIT CHANGES;"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Next, we could define our input data (input.json):","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"18jLZ0H5sie8iysm0AdBWu","type":"Entry","createdAt":"2023-01-18T18:00:16.499Z","updatedAt":"2023-01-18T18:00:16.499Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"json","code":"{\n \"inputs\": [\n {\n \"topic\": \"geo_events\",\n \"timestamp\": 1647515491,\n \"value\": {\n \"TIMESTAMP\": \"1647515491\",\n \"DEVICEID\": \"1234-e4b6-5678\",\n \"LATLONG\": \"38.8951,-77.0364\"\n }\n },\n {\n \"topic\": \"geo_events\",\n \"timestamp\": 1647799800,\n \"value\": {\n \"TIMESTAMP\": \"1647799800\",\n \"DEVICEID\": \"9876-e1p3-6763\",\n \"LATLONG\": \"51.5072,0.1276\"\n }\n }\n ]\n}"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Lastly, the expected output of our records post-enrichment (output.json). This is inclusive of the enriched values that are added via the CASE statement in our 'statements.sql' file:","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4Kckelo1i5jDQy1hCuv9Gp","type":"Entry","createdAt":"2023-01-18T18:00:39.786Z","updatedAt":"2023-01-18T18:00:39.786Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"json","code":"{\n \"outputs\": [\n {\n \"topic\": \"GEO_EVENTS_ENRICHED\",\n \"timestamp\": 1647515491,\n \"value\": {\n \"TIMESTAMP\": \"1647515491\",\n \"DEVICEID\": \"1234-e4b6-5678\",\n \"LATLONG\": \"38.8951,-77.0364\",\n \"DRIVERNAME\": \"Joe Bloggs\"\n }\n },\n {\n \"topic\": \"GEO_EVENTS_ENRICHED\",\n \"timestamp\": 1647799800,\n \"value\": {\n \"TIMESTAMP\": \"1647799800\",\n \"DEVICEID\": \"9876-e1p3-6763\",\n \"LATLONG\": \"51.5072,0.1276\",\n \"DRIVERNAME\": \"Jane Doe\"\n }\n }\n ]\n}"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Finally, all that would be required is to execute the test via the ksqlDB test runner","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3cQkQDhti19ziYTwhrXsRk","type":"Entry","createdAt":"2023-01-18T18:00:56.063Z","updatedAt":"2023-01-18T18:00:56.063Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"terminalBlock"}},"locale":"en-US"},"fields":{"command":"ksql-test-runner -s statements.sql -i input.json -o output.json"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In the case of a failure, the terminal will provide insight into the actual output versus the expected output. However, in this case, our test is successful and the output will confirm this:","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5UMvAYL3V7QFBsORs9NpW1","type":"Entry","createdAt":"2023-01-18T18:01:11.351Z","updatedAt":"2023-01-18T18:01:11.351Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"terminalBlock"}},"locale":"en-US"},"fields":{"command":">>> Test passed!"}}},"content":[]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Testing ksqlDB via Postman","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Another option for testing ksqlDB applications is to use a REST client. ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.postman.com/"},"content":[{"nodeType":"text","value":"Postman","marks":[],"data":{}}]},{"nodeType":"text","value":" provides a GUI that reduces some of the complexities of interacting with the ksqlDB server. Using Postman, it’s possible to inspect stream metadata, select data, and insert data using SQL statements.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Assuming your ksqlDB server is started locally on port 8088, you can configure a POST request to the below endpoint. Note you may need to update the proxy settings in the Postman application. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"http://localhost:8088/ksql","marks":[{"type":"code"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Next, you can add your SQL statements in the ","marks":[],"data":{}},{"nodeType":"text","value":"body","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" of the request. Below demonstrates how to retrieve metadata about created streams.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"6qtuD0X78uefYYp9WMvrMW","type":"Asset","createdAt":"2022-03-24T16:32:41.882Z","updatedAt":"2022-09-25T13:34:13.741Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"How to retrieve metatdata about created streams","description":"Testing Apache Kafka, integration test Kafka","file":{"url":"//images.ctfassets.net/lm7q36eaz111/6qtuD0X78uefYYp9WMvrMW/dfad476c40c45109072529ef65464522/Screen_Shot_2022-03-24_at_11.32.33_AM.png","details":{"size":345102,"image":{"width":1048,"height":682}},"fileName":"Screen Shot 2022-03-24 at 11.32.33 AM.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Assuming we want to insert a record into our ","marks":[],"data":{}},{"nodeType":"text","value":"geo_events","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":" topic, we could amend the KSQL statement accordingly:","marks":[],"data":{}}]},{"nodeType":"embedded-entry-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5C1HMJsmwGB9vnOxR1ND83","type":"Entry","createdAt":"2023-01-18T18:02:14.324Z","updatedAt":"2023-01-18T18:02:14.324Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"codeblock"}},"locale":"en-US"},"fields":{"language":"json","code":"{\n \"ksql\": \"INSERT INTO GEO_EVENTS VALUES('1647515491', '1234-e4b6-5678', '38.8951,-77.0364');\",\n \"streamProperties\": {}\n}"}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"There are significant limitations to how far your testing efforts can go using ksqlDB and REST clients. The Postman sandbox is a JavaScript environment and is generally single-threaded (meaning it can only process one command at a time). These efforts are once again manual and better suited only for ad-hoc debugging. ","marks":[],"data":{}}]},{"nodeType":"hr","data":{},"content":[]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"Conclusion","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"As we have demonstrated, testing different components within our Apache Kafka architecture is more than possible. At times, it does feel like the process is manual, fragmented, or even incomplete. Not to mention it comes with a degree of learning and ramp-up to familiarise yourself with the different approaches.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"To build a robust testing culture, it’s important that business people, QA engineers, and developers work together. These minds think differently; while an engineer might be concerned about working code, a product manager is more likely to think about the end users’ experience. Only together can they ensure complete coverage of testable scenarios through appropriate business rules. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"At Conduktor, we think there is an opportunity to do better in this space. Our aim is to provide intelligent and complementary tooling that simplifies complex and mundane operational tasks relating to the Kafka ecosystem.\n\nThat’s why we’re working on a game-changing new product to supercharge test automation and quality management of Kafka-based architectures. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"If you'd like to see for yourself, ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/get-started/"},"content":[{"nodeType":"text","value":"get Conduktor for free","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]}]},"author":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"1JmKff2d8J5C93f4yUSfCV","type":"Entry","createdAt":"2021-12-08T13:19:31.889Z","updatedAt":"2021-12-08T13:19:31.889Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":1,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"person"}},"locale":"en-US"},"fields":{"name":"James","title":"Technical Product Manager","linkedIn":"https://www.linkedin.com/in/james-white-301b5550/","image":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"65GcHlPUFtafUImTScSJjv","type":"Asset","createdAt":"2021-12-08T13:18:31.985Z","updatedAt":"2022-09-25T14:00:19.941Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"james","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/65GcHlPUFtafUImTScSJjv/d3f2ce92a6a6e7c8b4abc37b3da6fafa/james.png","details":{"size":47354,"image":{"width":192,"height":192}},"fileName":"james.png","contentType":"image/png"}}}}},"publishDate":"2022-03-25T00:00-05:00"}},{"fields":{"isLocal":false,"seo":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"MaR5iGSKfONml5AZJOgPO","type":"Entry","createdAt":"2022-03-24T18:59:08.185Z","updatedAt":"2022-09-25T13:52:30.980Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"contentType":{"sys":{"type":"Link","linkType":"ContentType","id":"seo"}},"locale":"en-US"},"fields":{"title":"Managing your Kafka Topics with Conduktor","description":"Before we look at how Conduktor helps you manage the Topics on your Kafka cluster, let's first look at what a topic is in Apache Kafka. ","noIndex":false}},"slug":"managing-the-topics-on-your-kafka-cluster","title":"Managing your Kafka Topics with Conduktor","navbarDescription":"A look at how Conduktor helps you manage the Topics on your Kafka cluster.","description":"A look at how Conduktor helps you manage the Topics on your Kafka cluster.","cover":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"3Rt99iPoVw0MEVlEXe0RFJ","type":"Asset","createdAt":"2022-03-24T18:59:01.545Z","updatedAt":"2022-09-25T08:17:47.148Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":2,"locale":"en-US"},"fields":{"title":"markus-spiske-Skf7HxARcoc-unsplash","description":"","file":{"url":"//images.ctfassets.net/lm7q36eaz111/3Rt99iPoVw0MEVlEXe0RFJ/90c9095b8592da51f1a2426dfeb1ca53/markus-spiske-Skf7HxARcoc-unsplash.jpeg","details":{"size":4148074,"image":{"width":5760,"height":3840}},"fileName":"markus-spiske-Skf7HxARcoc-unsplash.jpeg","contentType":"image/jpeg"}}},"richText":{"nodeType":"document","data":{},"content":[{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"What is Apache Kafka and Conduktor?","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Before we look at how Conduktor helps you manage the topics on your Kafka cluster, let's first look at what a topic is in Apache Kafka. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Apache Kafka is a pub-sub system that is composed of a network of machines called brokers. The brokers accept “events” from event producers and retain them based on time or message size, allowing consumers to read and process them at will.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You can find lots more information about Apache Kafka on ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/"},"content":[{"nodeType":"text","value":"Conduktor Kafkademy","marks":[],"data":{}}]},{"nodeType":"text","value":", our free learning resource.","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"What is a Topic?","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"5pNdzkCQcwC1RsKoglkI1Z","type":"Asset","createdAt":"2022-03-24T19:00:52.913Z","updatedAt":"2022-09-25T08:17:51.510Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"A topic in Apache Kafka","description":"What is a topic?","file":{"url":"//images.ctfassets.net/lm7q36eaz111/5pNdzkCQcwC1RsKoglkI1Z/394673b94ce4c3ccded62515b0e8cd5d/image6.png","details":{"size":51105,"image":{"width":957,"height":289}},"fileName":"image6.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A topic in Apache Kafka can be compared to a table in a relational database or a folder in a file system. Topics can additionally be broken into several partitions, which we will go into more detail later and you can also find more information ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/kafka-topics/"},"content":[{"nodeType":"text","value":"here","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":". ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A topic keeps a log of all messages you send to it and is recognised by the topic name. The topic naming strategy can be very specific to your own set up with a number of factors you can incorporate, including, where your data is coming from, how many topics you plan to have and the environment. You can find some interesting links for topic naming strategies ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://medium.com/@kiranprabhu/kafka-topic-naming-conventions-best-practices-6b6b332769a3"},"content":[{"nodeType":"text","value":"here","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" and ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://cnr.sh/essays/how-paint-bike-shed-kafka-topic-naming-conventions"},"content":[{"nodeType":"text","value":"here","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" which may be useful.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In summary, topics are the high-level structure that holds events together:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"They are comparable to a table or a folder.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Can have a specific naming strategy.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Can house many partitions, as discussed below.","marks":[],"data":{}}]}]}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"Partitions","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"9A44Vu0L11pKoyEJmfmW3","type":"Asset","createdAt":"2022-03-24T19:01:44.488Z","updatedAt":"2022-09-25T08:17:55.749Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":4,"locale":"en-US"},"fields":{"title":"Partitions on a Kafka","description":"Partitions","file":{"url":"//images.ctfassets.net/lm7q36eaz111/9A44Vu0L11pKoyEJmfmW3/49432cc0ac502a30b4ac2adae7ced5dd/image9.png","details":{"size":136606,"image":{"width":1109,"height":461}},"fileName":"image9.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Topics are typically broken down into multiple parts or “","marks":[],"data":{}},{"nodeType":"text","value":"partitions","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"”. A single topic may have more than one partition and it is common to see topics with multiple partitions. Shown above is a topic with three partitions.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Partitions are the way that Kafka ensures scalability, elasticity and redundancy across a number of brokers. Each partition can be hosted across multiple brokers to increase performance and ensure redundancy if a broker were to fail. ","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"The number of partitions of a topic is specified at the time of creation; however, Conduktor allows you to easily add new partitions. For more in-depth information on partitioning, check out this page on ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/kafka/kafka-topics/"},"content":[{"nodeType":"text","value":"Conduktor Kafkademy","marks":[],"data":{}}]},{"nodeType":"text","value":".","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"In summary, partitions are the smallest storage unit that holds a subset of events owned by a topic:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"A new event always goes on the end.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"They can only be read by seeking to offset and scanning forward from there.","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Events are immutable.","marks":[],"data":{}}]}]}]},{"nodeType":"heading-2","data":{},"content":[{"nodeType":"text","value":"How Conduktor improves Topic Management?","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Understanding what topics are and how they function in Apache Kafka is one thing but how can Conduktor help you manage your topics?","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor helps you in three major ways: ","marks":[],"data":{}}]},{"nodeType":"ordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Navigation","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Cluster Health","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Cluster Administration","marks":[],"data":{}}]}]}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"1. Navigation","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Navigating your topics and finding the one you are looking for can be a time-consuming process especially if you have 50, 100 or even 1,000 topics. Conduktor makes this process easier for you in a number of ways. Firstly, with a graphical view of all topics in your cluster, condensing the topic name through a feature called “","marks":[],"data":{}},{"nodeType":"text","value":"Smart Groups","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"”, tagging certain topics as “","marks":[],"data":{}},{"nodeType":"text","value":"Favourites","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"” and illustrating how recently active a topic has been.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Smart Groups","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Let's say you have a problem in production and you want to carry out some data inspection on a particular topic. When there are dozens to potentially hundreds of topics involved and depending on your topic naming strategy this can be easier said than done. “","marks":[],"data":{}},{"nodeType":"text","value":"Smart Groups","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"” in Conduktor can make your life much simpler in this regard.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"2f7pSkh6OAEYMzmfmkvojc","type":"Asset","createdAt":"2022-03-24T19:03:52.962Z","updatedAt":"2022-09-25T08:18:01.147Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":5,"locale":"en-US"},"fields":{"title":"Smart Groups Feature is Off","description":"Smart groups feature is off, apache kafka topics","file":{"url":"//images.ctfassets.net/lm7q36eaz111/2f7pSkh6OAEYMzmfmkvojc/aaf37b94fbf2bfe9e96a772b7e14d1a1/image2.png","details":{"size":401215,"image":{"width":1999,"height":1172}},"fileName":"image2.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"By using “","marks":[],"data":{}},{"nodeType":"text","value":"Smart Groups","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"” you can save time when trying to navigate the topics on your cluster and provide some more information such as the “","marks":[],"data":{}},{"nodeType":"text","value":"area","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"” and “","marks":[],"data":{}},{"nodeType":"text","value":"environment","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"” of each topic in this case.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"You configure through a regular expression (regex) your naming strategy or multiple naming strategies and then turn on or off with the click of a button.","marks":[],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"For an in-depth look into the “","marks":[],"data":{}},{"nodeType":"text","value":"Smart Groups","marks":[{"type":"italic"}],"data":{}},{"nodeType":"text","value":"” feature please read our blog ","marks":[],"data":{}},{"nodeType":"hyperlink","data":{"uri":"https://www.conduktor.io/blog/conduktor-2-1-smart-groups-feature/"},"content":[{"nodeType":"text","value":"here. ","marks":[{"type":"underline"}],"data":{}}]},{"nodeType":"text","value":" ","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4xtm88lBjX1KvmujZSyLZm","type":"Asset","createdAt":"2022-03-24T19:05:10.755Z","updatedAt":"2022-09-25T08:18:04.833Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Smart Groups Feature is On","description":"Smart groups feature is on, apache kafka topics","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4xtm88lBjX1KvmujZSyLZm/2413719b5baf138343cfda687aef647f/image1.png","details":{"size":371750,"image":{"width":1999,"height":1172}},"fileName":"image1.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Favourites","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Maybe you have a few topics that you always use or just a few that are always needed to carry out some troubleshooting. You can save yourself a lot of time by adding a favourite tag to them in Conduktor. They will have a blue star beside them so that you can filter them out from the rest to keep track of these specific topics.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"48Ba2lKkfGtQNx4m4Qi7rj","type":"Asset","createdAt":"2022-03-24T19:06:35.256Z","updatedAt":"2022-09-25T08:18:09.676Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Favourites Tagged Topics Only","description":"Apache Kafka topics, favourites tagged","file":{"url":"//images.ctfassets.net/lm7q36eaz111/48Ba2lKkfGtQNx4m4Qi7rj/c728d48891a7332498664524bcaee614/image3.png","details":{"size":194634,"image":{"width":1999,"height":1172}},"fileName":"image3.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Recent Activity ","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"It can be difficult to know which topics you use most regularly or even which topics you can get rid of if you are running low on disk space. The activity tab in Conduktor helps you understand the topics that are regularly used. So you can easily add them to the favourites and remove any that are not regularly used.","marks":[],"data":{}}]},{"nodeType":"embedded-asset-block","data":{"target":{"metadata":{"tags":[]},"sys":{"space":{"sys":{"type":"Link","linkType":"Space","id":"lm7q36eaz111"}},"id":"4H4uPJBtp4hdKCxHVlUCeI","type":"Asset","createdAt":"2022-03-24T19:07:21.541Z","updatedAt":"2022-09-25T08:18:14.898Z","environment":{"sys":{"id":"master","type":"Link","linkType":"Environment"}},"revision":3,"locale":"en-US"},"fields":{"title":"Recent Activity Tab","description":"Apache Kafka recent activity tab","file":{"url":"//images.ctfassets.net/lm7q36eaz111/4H4uPJBtp4hdKCxHVlUCeI/ae7fc0e4023e16b2b0c48bb1491b8f7d/Topic_List.png","details":{"size":365213,"image":{"width":1999,"height":1172}},"fileName":"Topic List.png","contentType":"image/png"}}}},"content":[]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"","marks":[],"data":{}}]},{"nodeType":"heading-3","data":{},"content":[{"nodeType":"text","value":"2. Cluster Health","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Overall Cluster Level","marks":[{"type":"bold"}],"data":{}}]},{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Conduktor provides a view of all topics on your cluster, including:","marks":[],"data":{}}]},{"nodeType":"unordered-list","data":{},"content":[{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Overall number of partitions","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Under Replicated Partitions (URP), ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Partitions without a Leader (No Leader) ","marks":[],"data":{}}]}]},{"nodeType":"list-item","data":{},"content":[{"nodeType":"paragraph","data":{},"content":[{"nodeType":"text","value":"Partitions that are below the required minimum in sync replicas (