/*@jsxRuntime automatic @jsxImportSource react*/ const {Fragment: _Fragment, jsx: _jsx, jsxs: _jsxs} = arguments[0]; function MDXContent(props = {}) { const {wrapper: MDXLayout} = props.components || ({}); return MDXLayout ? _jsx(MDXLayout, Object.assign({}, props, { children: _jsx(_createMdxContent, {}) })) : _createMdxContent(); function _createMdxContent() { const _components = Object.assign({ p: "p", ul: "ul", li: "li", a: "a", h2: "h2", code: "code", blockquote: "blockquote", strong: "strong", ol: "ol", span: "span", math: "math", semantics: "semantics", mrow: "mrow", mi: "mi", annotation: "annotation", em: "em", h3: "h3", pre: "pre", msup: "msup", mo: "mo" }, props.components); return _jsxs(_Fragment, { children: [_jsx(_components.p, { children: "In a previous post we set-up the OpenAI Gym to interface with our Javascript environment. Let's now look at how we can use this interface to run the CartPole example and solve it with the theory that we learned in previous blog posts." }), "\n", _jsx(_components.p, { children: "Requirements:" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "/post/2018/5/1/ai-ml-rl-rl-intro", children: "An introduction to Reinforcement Learning (RL)" }) }), "\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "/post/2018/5/20/ai-ml-rl-markov-property-chain-reward-decision", children: "The Markov Property, Chain, Reward Process and Decision Process" }) }), "\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "/post/2018/5/25/ai-ml-rl-bellman-equations", children: "Bellman Equations" }) }), "\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "/post/2018/5/7/ai-ml-rl-running-openai-gym-on-windows-and-js", children: "How to run OpenAI Gym on Windows and with Javascript" }) }), "\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "/post/2018/6/7/coding-javascript-rl-creating-a-bucketing-function", children: "Bucketization" }) }), "\n"] }), "\n", _jsx(_components.h2, { children: "Setting up our Experiment" }), "\n", _jsxs(_components.p, { children: ["We are again going to use Javascript to solve this, so everything you did before in the first article in our requirements comes in handy. Start by creating a new directory with our ", _jsx(_components.code, { children: "package.json" }), " and a ", _jsx(_components.code, { children: "index.js" }), " file for our main entry point."] }), "\n", _jsxs(_components.blockquote, { children: ["\n", _jsxs(_components.p, { children: [_jsx(_components.strong, { children: "Note:" }), " To make things easier, it's recommended to add a line that will start our server in the ", _jsx(_components.code, { children: "package.json" }), " file under the ", _jsx(_components.code, { children: "scripts" }), ": ", _jsx(_components.code, { children: "nodemon -x 'python ../OpenAI-Gym/server.py'" }), ". This way we can run ", _jsx(_components.code, { children: "npm run server" }), " to start our server, just make sure that you also run ", _jsx(_components.code, { children: "npm install nodemon --save-dev" }), " to ensure that the package is installed. Make sure to edit the path to point to the correct directories (in my case I put my experiments just one directory above the OpenAI-Gym files)"] }), "\n"] }), "\n", _jsxs(_components.p, { children: ["In the newly created ", _jsx(_components.code, { children: "index.js" }), " file we can now write some boilerplate code that will allow us to run our environment and visualize it. See the bottom of this article for the contents of this file."] }), "\n", _jsx(_components.h2, { children: "Getting to know the different methods" }), "\n", _jsx(_components.p, { children: "We learned the basics of RL in a previous article (see requirements) where we learned the following cycle:" }), "\n", _jsxs(_components.ol, { children: ["\n", _jsxs(_components.li, { children: ["We get an initial observation from an environment (the world) which we call our state ", _jsx(_components.span, { className: "math math-inline", children: _jsxs(_components.span, { className: "katex", children: [_jsx(_components.span, { className: "katex-mathml", children: _jsx(_components.math, { xmlns: "http://www.w3.org/1998/Math/MathML", children: _jsxs(_components.semantics, { children: [_jsx(_components.mrow, { children: _jsx(_components.mi, { children: "s" }) }), _jsx(_components.annotation, { encoding: "application/x-tex", children: "s" })] }) }) }), _jsx(_components.span, { className: "katex-html", "aria-hidden": "true", children: _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.4306em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "s" })] }) })] }) })] }), "\n", _jsxs(_components.li, { children: ["Based on this state ", _jsx(_components.span, { className: "math math-inline", children: _jsxs(_components.span, { className: "katex", children: [_jsx(_components.span, { className: "katex-mathml", children: _jsx(_components.math, { xmlns: "http://www.w3.org/1998/Math/MathML", children: _jsxs(_components.semantics, { children: [_jsx(_components.mrow, { children: _jsx(_components.mi, { children: "s" }) }), _jsx(_components.annotation, { encoding: "application/x-tex", children: "s" })] }) }) }), _jsx(_components.span, { className: "katex-html", "aria-hidden": "true", children: _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.4306em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "s" })] }) })] }) }), ", our agent will choose an action ", _jsx(_components.span, { className: "math math-inline", children: _jsxs(_components.span, { className: "katex", children: [_jsx(_components.span, { className: "katex-mathml", children: _jsx(_components.math, { xmlns: "http://www.w3.org/1998/Math/MathML", children: _jsxs(_components.semantics, { children: [_jsx(_components.mrow, { children: _jsx(_components.mi, { children: "a" }) }), _jsx(_components.annotation, { encoding: "application/x-tex", children: "a" })] }) }) }), _jsx(_components.span, { className: "katex-html", "aria-hidden": "true", children: _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.4306em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "a" })] }) })] }) })] }), "\n", _jsx(_components.li, { children: "This action will modify our world, and receive a reward based on the output of it" }), "\n", _jsx(_components.li, { children: "We go back to step 2" }), "\n"] }), "\n", _jsx(_components.p, { children: "These steps are implemented in OpenAI through the following methods:" }), "\n", _jsxs(_components.ol, { children: ["\n", _jsxs(_components.li, { children: ["Get our initial observation: ", _jsx(_components.code, { children: "state.observation = await client.envReset(instanceId)" })] }), "\n", _jsxs(_components.li, { children: ["Now decide which action to take based on the observation and the ", _jsx(_components.em, { children: "previous" }), " reward: ", _jsx(_components.code, { children: "agent.chooseAction(state.observation, state.reward, state.done)" })] }), "\n", _jsxs(_components.li, { children: ["Get a new observation from our world: ", _jsx(_components.code, { children: "state = await client.envStep(instanceId, actionToTake, true);" })] }), "\n", _jsx(_components.li, { children: "We go back to step 2" }), "\n"] }), "\n", _jsx(_components.p, { children: "To be able to make everything dynamic and create a true self-learning algorithm, OpenAI provides several other methods as well:" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["Create our environment: ", _jsx(_components.code, { children: "await client.envCreate(envId);" })] }), "\n", _jsxs(_components.li, { children: ["Get our actionSpace containing the different actions that we can take", "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Example:" }), " For ", _jsx(_components.code, { children: "CartPole-v0" }), " this returns: ", _jsx(_components.code, { children: "{ info: { n: 2, name: 'Discrete' } }" })] }), "\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Method:" }), " ", _jsx(_components.code, { children: "await client.envActionSpaceInfo(instanceId);" })] }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["Get our observationSpace containing the different observations that will be returned", "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Example:" }), " For ", _jsx(_components.code, { children: "CartPole-v0" }), " this returns: ", _jsx(_components.code, { children: "{ info: { high: [], low: [], name: 'Box', shape: [ 4 ] } }" })] }), "\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Method:" }), " ", _jsx(_components.code, { children: "await client.envObservationSpaceInfo(instanceId);" })] }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["Starting and closing the monitoring process: ", _jsx(_components.code, { children: "await client.envMonitorStart(instanceId, outDir, true);" }), ", ", _jsx(_components.code, { children: "await client.envMonitorClose(instanceId);" })] }), "\n", _jsxs(_components.li, { children: ["Upload the results to view it online: ", _jsx(_components.code, { children: "await client.upload(outDir)" })] }), "\n"] }), "\n", _jsx(_components.h2, { children: "Solving the CartPole problem" }), "\n", _jsx(_components.h3, { children: "Discovering the inputs" }), "\n", _jsx(_components.p, { children: "So in our CartPole example, let's take a look at one of our state observations:" }), "\n", _jsx(_components.pre, { className: "language-json", children: _jsxs(_components.code, { className: "language-json code-highlight", children: [_jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" done", _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token boolean", children: "false" }), _jsx(_components.span, { className: "token punctuation", children: "," }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" info", _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), _jsx(_components.span, { className: "token punctuation", children: "}" }), _jsx(_components.span, { className: "token punctuation", children: "," }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" observation", _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token punctuation", children: "[" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token number", children: "0.10040040129814863" }), _jsx(_components.span, { className: "token punctuation", children: "," }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token number", children: "0.5559659866710018" }), _jsx(_components.span, { className: "token punctuation", children: "," }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token number", children: "-0.17649755837025902" }), _jsx(_components.span, { className: "token punctuation", children: "," }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token number", children: "-1.1301794535568273" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "," }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" reward", _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token number", children: "1" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] })] }) }), "\n", _jsxs(_components.p, { children: ["Basically, these numbers are 4 unknown random continuous numbers, that we do not know what they do in our algorithm... (though we can make an interpretation of them with our human brain and say that they represent ", _jsxs(_components.strong, { children: ["position ", _jsx(_components.span, { className: "math math-inline", children: _jsxs(_components.span, { className: "katex", children: [_jsx(_components.span, { className: "katex-mathml", children: _jsx(_components.math, { xmlns: "http://www.w3.org/1998/Math/MathML", children: _jsxs(_components.semantics, { children: [_jsx(_components.mrow, { children: _jsx(_components.mi, { children: "x" }) }), _jsx(_components.annotation, { encoding: "application/x-tex", children: "x" })] }) }) }), _jsx(_components.span, { className: "katex-html", "aria-hidden": "true", children: _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.4306em" } }), _jsx(_components.span, { className: "mord mathnormal", children: "x" })] }) })] }) })] }), ", ", _jsxs(_components.strong, { children: ["velocity ", _jsx(_components.span, { className: "math math-inline", children: _jsxs(_components.span, { className: "katex", children: [_jsx(_components.span, { className: "katex-mathml", children: _jsx(_components.math, { xmlns: "http://www.w3.org/1998/Math/MathML", children: _jsxs(_components.semantics, { children: [_jsx(_components.mrow, { children: _jsx(_components.mi, { children: "v" }) }), _jsx(_components.annotation, { encoding: "application/x-tex", children: "v" })] }) }) }), _jsx(_components.span, { className: "katex-html", "aria-hidden": "true", children: _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.4306em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.03588em" }, children: "v" })] }) })] }) })] }), ", ", _jsxs(_components.strong, { children: ["angle ", _jsx(_components.span, { className: "math math-inline", children: _jsxs(_components.span, { className: "katex", children: [_jsx(_components.span, { className: "katex-mathml", children: _jsx(_components.math, { xmlns: "http://www.w3.org/1998/Math/MathML", children: _jsxs(_components.semantics, { children: [_jsx(_components.mrow, { children: _jsx(_components.mi, { children: "θ" }) }), _jsx(_components.annotation, { encoding: "application/x-tex", children: "\\theta" })] }) }) }), _jsx(_components.span, { className: "katex-html", "aria-hidden": "true", children: _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.6944em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.02778em" }, children: "θ" })] }) })] }) })] }), " and ", _jsxs(_components.strong, { children: ["angular velocity ", _jsx(_components.span, { className: "math math-inline", children: _jsxs(_components.span, { className: "katex", children: [_jsx(_components.span, { className: "katex-mathml", children: _jsx(_components.math, { xmlns: "http://www.w3.org/1998/Math/MathML", children: _jsxs(_components.semantics, { children: [_jsx(_components.mrow, { children: _jsx(_components.mi, { children: "α" }) }), _jsx(_components.annotation, { encoding: "application/x-tex", children: "\\alpha" })] }) }) }), _jsx(_components.span, { className: "katex-html", "aria-hidden": "true", children: _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.4306em" } }), _jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.0037em" }, children: "α" })] }) })] }) })] }), ")."] }), "\n", _jsx(_components.p, { children: "Next to these 4 random numbers, we also know by observing the action space that we can take 2 specific discrete actions (again by using our human brain we can classify this as moving the cart to the left or moving it to the right)." }), "\n", _jsx(_components.h3, { children: "Solving the problem" }), "\n", _jsxs(_components.p, { children: ["Applying what we learned in the previous article of [RL Intro][/post/2018/5/1/ai-ml-rl-rl-intro], we know that we now need to create a policy that shows us which state we will land in after taking a certain action. By then combining this with the ", _jsx(_components.a, { href: "/post/2018/5/25/ai-ml-rl-bellman-equations", children: "Bellman Equations" }), ", we are able to create an optimal ", _jsx(_components.span, { className: "math math-inline", children: _jsxs(_components.span, { className: "katex", children: [_jsx(_components.span, { className: "katex-mathml", children: _jsx(_components.math, { xmlns: "http://www.w3.org/1998/Math/MathML", children: _jsxs(_components.semantics, { children: [_jsx(_components.mrow, { children: _jsxs(_components.msup, { children: [_jsx(_components.mi, { children: "π" }), _jsx(_components.mo, { lspace: "0em", rspace: "0em", children: "∗" })] }) }), _jsx(_components.annotation, { encoding: "application/x-tex", children: "\\pi^{*}" })] }) }) }), _jsx(_components.span, { className: "katex-html", "aria-hidden": "true", children: _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.6887em" } }), _jsxs(_components.span, { className: "mord", children: [_jsx(_components.span, { className: "mord mathnormal", style: { marginRight: "0.03588em" }, children: "π" }), _jsx(_components.span, { className: "msupsub", children: _jsx(_components.span, { className: "vlist-t", children: _jsx(_components.span, { className: "vlist-r", children: _jsx(_components.span, { className: "vlist", style: { height: "0.6887em" }, children: _jsxs(_components.span, { style: { top: "-3.063em", marginRight: "0.05em" }, children: [_jsx(_components.span, { className: "pstrut", style: { height: "2.7em" } }), _jsx(_components.span, { className: "sizing reset-size6 size3 mtight", children: _jsx(_components.span, { className: "mord mtight", children: _jsx(_components.span, { className: "mord mtight", children: "∗" }) }) })] }) }) }) }) })] })] }) })] }) }), " that will maximize the return in every state. So we follow our boilerplate of every reinforcement algorithm, stating that we first check our environment, then take an action and based on the reward of that action update our q-function. Or summarized:"] }), "\n", _jsxs(_components.ol, { children: ["\n", _jsx(_components.li, { children: "Observe the environment" }), "\n", _jsx(_components.li, { children: "Based on the observed environment, take an action" }), "\n", _jsx(_components.li, { children: "Now look at the reward that we got from taking an action, and update our Q-Table with this reward." }), "\n"] }), "\n", _jsxs(_components.p, { children: ["To start with point 1, we want to be able to represent our observation as a state index. This is needed to be able to construct our Q-Table which will consist of our state and actions. For this we use a method called ", _jsx(_components.a, { href: "/post/2018/6/7/coding-javascript-rl-creating-a-bucketing-function", children: "Bucketization" }), " which will convert our observations from a continuous state (the numbers that fit between our lower bound and upper bound can be described as ]", _jsx(_components.span, { className: "math math-inline", children: _jsxs(_components.span, { className: "katex", children: [_jsx(_components.span, { className: "katex-mathml", children: _jsx(_components.math, { xmlns: "http://www.w3.org/1998/Math/MathML", children: _jsxs(_components.semantics, { children: [_jsxs(_components.mrow, { children: [_jsx(_components.mo, { children: "−" }), _jsx(_components.mi, { mathvariant: "normal", children: "∞" })] }), _jsx(_components.annotation, { encoding: "application/x-tex", children: "-\\infty" })] }) }) }), _jsx(_components.span, { className: "katex-html", "aria-hidden": "true", children: _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.6667em", verticalAlign: "-0.0833em" } }), _jsx(_components.span, { className: "mord", children: "−" }), _jsx(_components.span, { className: "mord", children: "∞" })] }) })] }) }), ", ", _jsx(_components.span, { className: "math math-inline", children: _jsxs(_components.span, { className: "katex", children: [_jsx(_components.span, { className: "katex-mathml", children: _jsx(_components.math, { xmlns: "http://www.w3.org/1998/Math/MathML", children: _jsxs(_components.semantics, { children: [_jsx(_components.mrow, { children: _jsx(_components.mi, { mathvariant: "normal", children: "∞" }) }), _jsx(_components.annotation, { encoding: "application/x-tex", children: "\\infty" })] }) }) }), _jsx(_components.span, { className: "katex-html", "aria-hidden": "true", children: _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.4306em" } }), _jsx(_components.span, { className: "mord", children: "∞" })] }) })] }) }), "[) towards a discrete state (we \"bucket\" our numbers, returning only a subset of indexes rather than ", _jsx(_components.span, { className: "math math-inline", children: _jsxs(_components.span, { className: "katex", children: [_jsx(_components.span, { className: "katex-mathml", children: _jsx(_components.math, { xmlns: "http://www.w3.org/1998/Math/MathML", children: _jsxs(_components.semantics, { children: [_jsx(_components.mrow, { children: _jsx(_components.mi, { mathvariant: "normal", children: "∞" }) }), _jsx(_components.annotation, { encoding: "application/x-tex", children: "\\infty" })] }) }) }), _jsx(_components.span, { className: "katex-html", "aria-hidden": "true", children: _jsxs(_components.span, { className: "base", children: [_jsx(_components.span, { className: "strut", style: { height: "0.4306em" } }), _jsx(_components.span, { className: "mord", children: "∞" })] }) })] }) }), " ones), allowing us to keep our states minimal and represent them in our memory."] }), "\n", _jsxs(_components.blockquote, { children: ["\n", _jsx(_components.p, { children: "Note: I did limit the velocity speed and the angle of our cartpole to achieve convergence faster." }), "\n"] }), "\n", _jsxs(_components.p, { children: ["For point 2, we will then use what we learned in the ", _jsx(_components.a, { href: "/post/2018/5/3/ai-ml-rl-n-arms-bandit-framework", children: "Multi-armed bandit framework" }), " article, stating that we take our actions as returned by taking the maximum value for that state it's actions (", _jsx(_components.code, { children: "Math.max(Q[s]" }), ") but that we add an ", _jsx(_components.strong, { children: "exploration factor" }), " that will still take a random action on some points, but still minimizes regret."] }), "\n", _jsx(_components.p, { children: "And as our last point 3, we then update our Q-Table by utilizing the Bellman Equation which results in the single line" }), "\n", _jsx(_components.pre, { className: "language-javascript", children: _jsx(_components.code, { className: "language-javascript code-highlight", children: _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "q" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "oldStateIdx", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "action", _jsx(_components.span, { className: "token punctuation", children: "]" }), " ", _jsx(_components.span, { className: "token operator", children: "+=" }), " alpha ", _jsx(_components.span, { className: "token operator", children: "*" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), "reward ", _jsx(_components.span, { className: "token operator", children: "+" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "gamma" }), " ", _jsx(_components.span, { className: "token operator", children: "*" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "q" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "newStateIdx", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "getMaxValue" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token operator", children: "-" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "q" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "oldStateIdx", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "action", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }) }) }), "\n", _jsx(_components.p, { children: "Successfully coding this and applying, will give us the following result:" }), "\n", _jsx(_components.pre, { className: "language-html", children: _jsxs(_components.code, { className: "language-html code-highlight", children: [_jsxs(_components.span, { className: "code-line", children: [_jsxs(_components.span, { className: "token tag", children: [_jsxs(_components.span, { className: "token tag", children: [_jsx(_components.span, { className: "token punctuation", children: "<" }), "video"] }), " ", _jsx(_components.span, { className: "token attr-name", children: "width" }), _jsxs(_components.span, { className: "token attr-value", children: [_jsx(_components.span, { className: "token punctuation attr-equals", children: "=" }), _jsx(_components.span, { className: "token punctuation", children: "\"" }), "640", _jsx(_components.span, { className: "token punctuation", children: "\"" })] }), " ", _jsx(_components.span, { className: "token attr-name", children: "height" }), _jsxs(_components.span, { className: "token attr-value", children: [_jsx(_components.span, { className: "token punctuation attr-equals", children: "=" }), _jsx(_components.span, { className: "token punctuation", children: "\"" }), "480", _jsx(_components.span, { className: "token punctuation", children: "\"" })] }), " ", _jsx(_components.span, { className: "token attr-name", children: "autoplay" }), _jsx(_components.span, { className: "token punctuation", children: ">" })] }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsxs(_components.span, { className: "token tag", children: [_jsxs(_components.span, { className: "token tag", children: [_jsx(_components.span, { className: "token punctuation", children: "<" }), "source"] }), " ", _jsx(_components.span, { className: "token attr-name", children: "src" }), _jsxs(_components.span, { className: "token attr-value", children: [_jsx(_components.span, { className: "token punctuation attr-equals", children: "=" }), _jsx(_components.span, { className: "token punctuation", children: "\"" }), "/videos/cartpole.mp4", _jsx(_components.span, { className: "token punctuation", children: "\"" })] }), " ", _jsx(_components.span, { className: "token attr-name", children: "type" }), _jsxs(_components.span, { className: "token attr-value", children: [_jsx(_components.span, { className: "token punctuation attr-equals", children: "=" }), _jsx(_components.span, { className: "token punctuation", children: "\"" }), "video/mp4", _jsx(_components.span, { className: "token punctuation", children: "\"" })] }), _jsx(_components.span, { className: "token punctuation", children: ">" })] }), "\n"] }), _jsx(_components.span, { className: "code-line", children: " Your browser does not support the video tag.\n" }), _jsxs(_components.span, { className: "code-line", children: [_jsxs(_components.span, { className: "token tag", children: [_jsxs(_components.span, { className: "token tag", children: [_jsx(_components.span, { className: "token punctuation", children: "" })] }), " \n"] })] }) }), "\n", _jsx(_components.h2, { children: "Files" }), "\n", _jsx(_components.h3, { children: "Index.js Template" }), "\n", _jsx(_components.pre, { className: "language-javascript", children: _jsxs(_components.code, { className: "language-javascript code-highlight", children: [_jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * For more information, check: https://gym.openai.com/docs/\n" }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "const" }), " gym ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token function", children: "require" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token string", children: "'../OpenAI-Gym/gym-http-api/binding-js/dist/lib/gymHTTPClient'" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "const" }), " ", _jsx(_components.span, { className: "token maybe-class-name", children: "Bucket" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token function", children: "require" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token string", children: "'./Bucket'" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "const" }), " episodeCount ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "1000" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "const" }), " maxTimeSteps ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "250" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "const" }), " outDir ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token string", children: "\"/tmp/random-agent/results\"" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "const" }), " numTrials ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "3" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "const" }), " envId ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token string", children: "\"CartPole-v0\"" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "const" }), " client ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "new" }), " ", _jsxs(_components.span, { className: "token class-name", children: ["gym", _jsx(_components.span, { className: "token punctuation", children: "." }), "default"] }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token string", children: "\"http://127.0.0.1:5000\"" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token comment", children: "// Parameter to enable debug" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "const" }), " ", _jsx(_components.span, { className: "token constant", children: "PRINT_DEBUG" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token boolean", children: "true" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "class" }), " ", _jsx(_components.span, { className: "token class-name", children: "Experiment" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "constructor" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token parameter", children: "agentClass" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "environment" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword null nil", children: "null" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "actionSpaceInfo" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword null nil", children: "null" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "agent" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword null nil", children: "null" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "async" }), " ", _jsx(_components.span, { className: "token function", children: "initialize" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// Create our environment" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "environment" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword control-flow", children: "await" }), " client", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "envCreate" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "envId", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// Our ActionSpace" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// For CartPole-v0 this returns: { info: { n: 2, name: 'Discrete' } }" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "actionSpaceInfo" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword control-flow", children: "await" }), " client", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "envActionSpaceInfo" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "environment" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "instance_id" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// Our ObservationSpace" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// For CartPole-v0 this returns: { info: { high: [XXX], low: [XXX], name: 'Box', shape: [ 4 ] } }" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "observationSpaceInfo" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword control-flow", children: "await" }), " client", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "envObservationSpaceInfo" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "environment" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "instance_id" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// Rebind our velocity and angular velocity parameters, the pole should stand still as much as possible" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "observationSpaceInfo" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "info" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "high" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token number", children: "1" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "0.5" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "observationSpaceInfo" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "info" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "low" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token number", children: "1" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token operator", children: "-" }), _jsx(_components.span, { className: "token number", children: "0.5" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "observationSpaceInfo" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "info" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "high" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token number", children: "3" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "radians" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token number", children: "50" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "observationSpaceInfo" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "info" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "low" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token number", children: "3" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token operator", children: "-" }), _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "radians" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token number", children: "50" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// Set our learning agent" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "agent" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "new" }), " ", _jsx(_components.span, { className: "token class-name", children: "CartPoleAgent" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "actionSpaceInfo" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "info" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token string", children: "\"n\"" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "observationSpaceInfo" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "info" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "shape" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token number", children: "0" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "observationSpaceInfo" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "info" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "async" }), " ", _jsx(_components.span, { className: "token function", children: "run" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " oldObservation ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "await" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "initialize" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// Start Monitoring" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "await" }), " client", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "envMonitorStart" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "environment" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "instance_id" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " outDir", _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token boolean", children: "true" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// For every episode" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "for" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "let" }), " episode ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "0" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " episode ", _jsx(_components.span, { className: "token operator", children: "<" }), " episodeCount", _jsx(_components.span, { className: "token punctuation", children: ";" }), " episode", _jsx(_components.span, { className: "token operator", children: "++" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// Reset the whole environment to start over" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " environment ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token literal-property property", children: "done" }), _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token boolean", children: "false" }), _jsx(_components.span, { className: "token punctuation", children: "," }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token literal-property property", children: "info" }), _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), _jsx(_components.span, { className: "token punctuation", children: "}" }), _jsx(_components.span, { className: "token punctuation", children: "," }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token literal-property property", children: "observation" }), _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token keyword control-flow", children: "await" }), " client", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "envReset" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "environment" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "instance_id" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token comment", children: "// envReset gives an initial observation" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token literal-property property", children: "reward" }), _jsx(_components.span, { className: "token operator", children: ":" }), " ", _jsx(_components.span, { className: "token number", children: "0" }), _jsx(_components.span, { className: "token punctuation", children: "," }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// Keep executing while we can:" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// if state.done = true, then that means the pole tipped to far, or we died" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// if t> =maxTimeSteps, then we did not solve it fast enough" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " t ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "0" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "while" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token operator", children: "!" }), "environment", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "done" }), " ", _jsx(_components.span, { className: "token operator", children: "&&" }), " t ", _jsx(_components.span, { className: "token operator", children: "<" }), " maxTimeSteps", _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" environment ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword control-flow", children: "await" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "agent" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "act" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "client", _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "environment" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "instance_id" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " episode", _jsx(_components.span, { className: "token punctuation", children: "," }), " environment", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "observation" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" t", _jsx(_components.span, { className: "token operator", children: "++" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token console class-name", children: "console" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "log" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsxs(_components.span, { className: "token template-string", children: [_jsx(_components.span, { className: "token template-punctuation string", children: "`" }), _jsx(_components.span, { className: "token string", children: "Episode " }), _jsxs(_components.span, { className: "token interpolation", children: [_jsx(_components.span, { className: "token interpolation-punctuation punctuation", children: "${" }), "episode", _jsx(_components.span, { className: "token interpolation-punctuation punctuation", children: "}" })] }), _jsx(_components.span, { className: "token string", children: " ended after " }), _jsxs(_components.span, { className: "token interpolation", children: [_jsx(_components.span, { className: "token interpolation-punctuation punctuation", children: "${" }), "t", _jsx(_components.span, { className: "token interpolation-punctuation punctuation", children: "}" })] }), _jsx(_components.span, { className: "token string", children: " timesteps" }), _jsx(_components.span, { className: "token template-punctuation string", children: "`" })] }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// Stop Monitoring" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "await" }), " client", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "envMonitorClose" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "environment" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "instance_id" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token console class-name", children: "console" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "log" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "outDir", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token comment", children: "// Create and run our experiment" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "const" }), " experiment ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "new" }), " ", _jsx(_components.span, { className: "token class-name", children: "Experiment" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: ["experiment", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "run" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token comment", children: "// Our Agent in charge of learning based on the observation and reward" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "class" }), " ", _jsx(_components.span, { className: "token class-name", children: "CartPoleAgent" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "constructor" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsxs(_components.span, { className: "token parameter", children: ["numberOfActions", _jsx(_components.span, { className: "token punctuation", children: "," }), " numberOfObservations", _jsx(_components.span, { className: "token punctuation", children: "," }), " observationSpaceInfo"] }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "numberOfActions" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " numberOfActions", _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "numberOfObservations" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " numberOfObservations", _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "observationSpaceInfo" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " observationSpaceInfo", _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "gamma" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "1.0" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " ", _jsx(_components.span, { className: "token comment", children: "// Discount factor" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "minEpsilon" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "0.1" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " ", _jsx(_components.span, { className: "token comment", children: "// Exploration factor" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "minAlpha" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "0.1" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " ", _jsx(_components.span, { className: "token comment", children: "// Learning Rate" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "adaDivisor" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "25" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// Create buckets and states, for the CartPole we know that it's x, v, theta, alpha (position, velocity, angle, angular velocity)" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "bucketArrays" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "initializeBuckets" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token number", children: "1" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token number", children: "1" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token number", children: "8" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token number", children: "10" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "states" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "initializeStates" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "bucketArrays" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "q" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "initializeQTable" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "states" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "length" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "numberOfActions" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "if" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token constant", children: "PRINT_DEBUG" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "printDebug" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "initializeStates" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token parameter", children: "buckets" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " bucketIndexes ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" buckets", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "forEach" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token parameter", children: "bucket" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token arrow operator", children: "=>" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" bucketIndexes", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "push" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "bucket", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "bucketIndexes" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " states ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token function", children: "cartesianProduct" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token spread operator", children: "..." }), "bucketIndexes", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "return" }), " states", _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" })] }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * We have an amount of observations, but to limit the states we can have we specify ranges or \"buckets\" for the values\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * In our initialization method, we define the amount of \"steps\" for each observation bucket \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Example: 1, 1, 6, 12 means that for observation #0 we set 1 bucket, #2 also 1, #3 6 buckets and #4 12 buckets\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * which represents that we will create 12 different steps for observation #4\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), "Array", _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "buckets" }), " \n"] }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "initializeBuckets" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token parameter", children: "bucketSizes" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " buckets ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "for" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "let" }), " i ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "0" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " i ", _jsx(_components.span, { className: "token operator", children: "<" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "numberOfObservations" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " i", _jsx(_components.span, { className: "token operator", children: "++" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "const" }), " bucketSize ", _jsx(_components.span, { className: "token operator", children: "=" }), " bucketSizes", _jsx(_components.span, { className: "token punctuation", children: "[" }), "i", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "const" }), " upperBound ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "observationSpaceInfo" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "high" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "i", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "const" }), " lowerBound ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "observationSpaceInfo" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "low" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "i", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" buckets", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "push" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "new" }), " ", _jsx(_components.span, { className: "token class-name", children: "Bucket" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "bucketSize", _jsx(_components.span, { className: "token punctuation", children: "," }), " lowerBound", _jsx(_components.span, { className: "token punctuation", children: "," }), " upperBound", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "return" }), " buckets", _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" })] }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Since the amount of spaces that we can have is huge, we need to limit this\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * This limiting is done through using \"buckets\", or we also say that we put everything that is between X and Y in bucket Z\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * For every feature we specify a range between where the values can be\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * TODO: Our Q-Table is [state][action] so we need to define the tuple which is our number of observations\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Therefore we need to \"compact\" them or write a \"range\" so that we do not overextend\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Note: this is also called \"discretization\"\n" }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "initializeQTable" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsxs(_components.span, { className: "token parameter", children: ["stateCount", _jsx(_components.span, { className: "token punctuation", children: "," }), " actionCount"] }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " q ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " ", _jsx(_components.span, { className: "token comment", children: "// Our Q-Table" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "for" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "let" }), " stateIdx ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "0" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " stateIdx ", _jsx(_components.span, { className: "token operator", children: "<" }), " stateCount", _jsx(_components.span, { className: "token punctuation", children: ";" }), " stateIdx", _jsx(_components.span, { className: "token operator", children: "++" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" q", _jsx(_components.span, { className: "token punctuation", children: "[" }), "stateIdx", _jsx(_components.span, { className: "token punctuation", children: "]" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "for" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "let" }), " actionIdx ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "0" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " actionIdx ", _jsx(_components.span, { className: "token operator", children: "<" }), " actionCount", _jsx(_components.span, { className: "token punctuation", children: ";" }), " actionIdx", _jsx(_components.span, { className: "token operator", children: "++" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" q", _jsx(_components.span, { className: "token punctuation", children: "[" }), "stateIdx", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "actionIdx", _jsx(_components.span, { className: "token punctuation", children: "]" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "0" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "return" }), " q", _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" })] }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Convert our observation to a state index based on the buckets\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), _jsx(_components.span, { className: "token operator", children: "*" }), _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "states" }), " \n"] }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), _jsx(_components.span, { className: "token operator", children: "*" }), _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "bucketArrays" }), " \n"] }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), _jsx(_components.span, { className: "token operator", children: "*" }), _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "observations" }), " \n"] }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "observationToStateIndex" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsxs(_components.span, { className: "token parameter", children: ["states", _jsx(_components.span, { className: "token punctuation", children: "," }), " bucketArrays", _jsx(_components.span, { className: "token punctuation", children: "," }), " observations"] }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " observationBucketIndexes ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "for" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "let" }), " i ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "0" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " i ", _jsx(_components.span, { className: "token operator", children: "<" }), " observations", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "length" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " i", _jsx(_components.span, { className: "token operator", children: "++" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" observationBucketIndexes", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "push" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "bucketArrays", _jsx(_components.span, { className: "token punctuation", children: "[" }), "i", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "getBucketIdxForValue" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "observations", _jsx(_components.span, { className: "token punctuation", children: "[" }), "i", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " stateIndex ", _jsx(_components.span, { className: "token operator", children: "=" }), " states", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "map" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token parameter", children: "i" }), " ", _jsx(_components.span, { className: "token arrow operator", children: "=>" }), " i", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "toString" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "indexOf" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "observationBucketIndexes", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "toString" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "return" }), " stateIndex", _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" })] }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), "Array", _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "observation" }), " array of observations (e.g. [1, 6, 9, 2.3])\n"] }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), "number", _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "reward" }), " The reward we got from taking the previous action\n"] }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "async" }), " ", _jsx(_components.span, { className: "token function", children: "act" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsxs(_components.span, { className: "token parameter", children: ["client", _jsx(_components.span, { className: "token punctuation", children: "," }), " instanceId", _jsx(_components.span, { className: "token punctuation", children: "," }), " episode", _jsx(_components.span, { className: "token punctuation", children: "," }), " observation"] }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// ========================================================" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// 1. Map the current observation to a state" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// ========================================================" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// We get a number of values back, convert these to states" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " stateIndex ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "observationToStateIndex" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "states" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "bucketArrays" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " observation", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// Learning discount factor and learning rate" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "const" }), " epsilon ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "getEpsilon" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "episode", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "const" }), " alpha ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "getAlpha" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "episode", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// ========================================================" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// 2. Based on our observation, choose and take an action and view the change in environment" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// ========================================================" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// Choose an action" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "const" }), " action ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "chooseAction" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "stateIndex", _jsx(_components.span, { className: "token punctuation", children: "," }), " epsilon", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " newEnvironment ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword control-flow", children: "await" }), " client", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "envStep" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "instanceId", _jsx(_components.span, { className: "token punctuation", children: "," }), " action", _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token boolean", children: "true" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " ", _jsx(_components.span, { className: "token comment", children: "// returns { done, info, observation, reward }" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: " \n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// ========================================================" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// 3. Based on result of the action, update our Q value" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// ========================================================" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " newStateIndex ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "observationToStateIndex" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "states" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "bucketArrays" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " newEnvironment", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "observation" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "updateQ" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "stateIndex", _jsx(_components.span, { className: "token punctuation", children: "," }), " action", _jsx(_components.span, { className: "token punctuation", children: "," }), " newEnvironment", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "reward" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " newStateIndex", _jsx(_components.span, { className: "token punctuation", children: "," }), " alpha", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "return" }), " newEnvironment", _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" })] }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Update our Q-Table based on the Bellman Equation for Action-Value\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), "number", _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "oldStateIdx" }), " \n"] }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), "number", _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "action" }), " \n"] }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), "number", _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "reward" }), " \n"] }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), "number", _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "newStateIdx" }), " \n"] }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), "number", _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "alpha" }), " \n"] }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "updateQ" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsxs(_components.span, { className: "token parameter", children: ["oldStateIdx", _jsx(_components.span, { className: "token punctuation", children: "," }), " action", _jsx(_components.span, { className: "token punctuation", children: "," }), " reward", _jsx(_components.span, { className: "token punctuation", children: "," }), " newStateIdx", _jsx(_components.span, { className: "token punctuation", children: "," }), " alpha"] }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "q" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "oldStateIdx", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "action", _jsx(_components.span, { className: "token punctuation", children: "]" }), " ", _jsx(_components.span, { className: "token operator", children: "+=" }), " alpha ", _jsx(_components.span, { className: "token operator", children: "*" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), "reward ", _jsx(_components.span, { className: "token operator", children: "+" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "gamma" }), " ", _jsx(_components.span, { className: "token operator", children: "*" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "q" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "newStateIdx", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "getMaxValue" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token operator", children: "-" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "q" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "oldStateIdx", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "action", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" })] }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * For exploration, return random action if Math.random< =explorationFactor (epsilon), else max Q value for the current state\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), "int", _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "state" }), " \n"] }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "chooseAction" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsxs(_components.span, { className: "token parameter", children: ["stateIndex", _jsx(_components.span, { className: "token punctuation", children: "," }), " epsilon"] }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " random ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "random" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " isRandom ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "random" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token operator", children: "<" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), "epsilon", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// If we want to explore (depending on epsilon), take a random action" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "if" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), "isRandom", _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "return" }), " ", _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "floor" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "random" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token operator", children: "*" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "numberOfActions" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "return" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "q" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "stateIndex", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "getIndexOfMaxValue" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" })] }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Our exploration factor\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * We use a dynamic parameter dependent on the episode, to be able to reduce exploration over our timeframe\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), _jsx(_components.span, { className: "token operator", children: "*" }), _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "t" }), " \n"] }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "getEpsilon" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token parameter", children: "t" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "return" }), " ", _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "max" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "minEpsilon" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "min" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token number", children: "1.0" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token number", children: "1.0" }), " ", _jsx(_components.span, { className: "token operator", children: "-" }), " ", _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "log10" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "t ", _jsx(_components.span, { className: "token operator", children: "+" }), " ", _jsx(_components.span, { className: "token number", children: "1" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token operator", children: "/" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "adaDivisor" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" })] }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Our Learning factor\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * We use a dynamic parameter dependent on the episode, to be able to reduce learning over our timeframe\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), _jsx(_components.span, { className: "token operator", children: "*" }), _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "t" }), " \n"] }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "getAlpha" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token parameter", children: "t" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "return" }), " ", _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "max" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "minAlpha" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "min" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token number", children: "0.5" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token number", children: "1.0" }), " ", _jsx(_components.span, { className: "token operator", children: "-" }), " ", _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "log10" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "t ", _jsx(_components.span, { className: "token operator", children: "+" }), " ", _jsx(_components.span, { className: "token number", children: "1" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token operator", children: "/" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "adaDivisor" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" })] }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * We get an observation (e.g.) [1, 6, 9, 2.3] which defines our \"state\" or what we currently see.\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * To get our state we need to look at each feature value (1, 6, ...) and define in which bucket these fit. Then we can resolve this to a state idx\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Our bucket id is then \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * In this method we define this to a unique entry for our Q-Table\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), _jsx(_components.span, { className: "token operator", children: "*" }), _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "observation" }), " \n"] }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "convertObservationToBucketIndexes" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token parameter", children: "observation" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " bucketIndexes ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "for" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "let" }), " i ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "0" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " i ", _jsx(_components.span, { className: "token operator", children: "<" }), " observation", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "length" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " i", _jsx(_components.span, { className: "token operator", children: "++" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " bucketIndex ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "floor" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "observation", _jsx(_components.span, { className: "token punctuation", children: "[" }), "i", _jsx(_components.span, { className: "token punctuation", children: "]" }), " ", _jsx(_components.span, { className: "token operator", children: "-" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "bucketArrays" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "i", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "lowerBound" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token operator", children: "/" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "bucketArrays" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "i", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "bucketSize" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" bucketIndexes", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "push" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "bucketIndex", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "return" }), " bucketIndexes", _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" })] }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Help method to view some details about our algorithm\n" }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "printDebug" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token console class-name", children: "console" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "log" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token string", children: "'======================================='" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token console class-name", children: "console" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "log" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token string", children: "'Buckets'" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token console class-name", children: "console" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "log" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token string", children: "'======================================='" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token console class-name", children: "console" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "table" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "bucketArrays" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token console class-name", children: "console" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "log" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token string", children: "'======================================='" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token console class-name", children: "console" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "log" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token string", children: "'Possible States'" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token console class-name", children: "console" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "log" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token string", children: "'======================================='" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token console class-name", children: "console" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "table" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "states" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token console class-name", children: "console" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "log" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token string", children: "'======================================='" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token console class-name", children: "console" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "log" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token string", children: "'Initial Q-Table'" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token console class-name", children: "console" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "log" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token string", children: "'======================================='" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token console class-name", children: "console" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "table" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "q" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Get the index value of the maximum value in an array\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * e.g. if our array is [2, 6, 3, 5] it will return 1 since 6 is the highest\n" }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token class-name", children: "Array" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "prototype" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method-variable function-variable method function property-access", children: "getIndexOfMaxValue" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "function" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "return" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "indexOf" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "max" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token spread operator", children: "..." }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Return the highest value in our array\n" }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token class-name", children: "Array" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "prototype" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method-variable function-variable method function property-access", children: "getMaxValue" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "function" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "return" }), " ", _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "max" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token spread operator", children: "..." }), _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Generate the cartesian product between sets\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * In other terms: generate all possible combinations of multiple arrays\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * https://stackoverflow.com/questions/12303989/cartesian-product-of-multiple-arrays-in-javascript\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * \n" }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), "Array", _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "inArr" }), " an array of numbers\n"] }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "const" }), " ", _jsx(_components.span, { className: "token function-variable function", children: "f" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsxs(_components.span, { className: "token parameter", children: ["a", _jsx(_components.span, { className: "token punctuation", children: "," }), " b"] }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token arrow operator", children: "=>" }), " ", _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "concat" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token spread operator", children: "..." }), "a", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "map" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token parameter", children: "d" }), " ", _jsx(_components.span, { className: "token arrow operator", children: "=>" }), " b", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "map" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token parameter", children: "e" }), " ", _jsx(_components.span, { className: "token arrow operator", children: "=>" }), " ", _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "concat" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "d", _jsx(_components.span, { className: "token punctuation", children: "," }), " e", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "const" }), " ", _jsx(_components.span, { className: "token function-variable function", children: "cartesianProduct" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsxs(_components.span, { className: "token parameter", children: ["a", _jsx(_components.span, { className: "token punctuation", children: "," }), " b", _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token spread operator", children: "..." }), "c"] }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token arrow operator", children: "=>" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), "b ", _jsx(_components.span, { className: "token operator", children: "?" }), " ", _jsx(_components.span, { className: "token function", children: "cartesianProduct" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token function", children: "f" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "a", _jsx(_components.span, { className: "token punctuation", children: "," }), " b", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: "," }), " ", _jsx(_components.span, { className: "token spread operator", children: "..." }), "c", _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token operator", children: ":" }), " a", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Convert degrees to radians\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), "number", _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "degrees" }), " \n"] }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method-variable function-variable method function property-access", children: "radians" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token parameter", children: "degrees" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token arrow operator", children: "=>" }), " degrees ", _jsx(_components.span, { className: "token operator", children: "*" }), " ", _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token constant", children: "PI" }), " ", _jsx(_components.span, { className: "token operator", children: "/" }), " ", _jsx(_components.span, { className: "token number", children: "180" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: "/**\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsx(_components.span, { className: "token doc-comment comment", children: " * Convert radians to degrees\n" }) }), _jsx(_components.span, { className: "code-line", children: _jsxs(_components.span, { className: "token doc-comment comment", children: [" * ", _jsx(_components.span, { className: "token keyword", children: "@param" }), " ", _jsxs(_components.span, { className: "token class-name", children: [_jsx(_components.span, { className: "token punctuation", children: "{" }), "number", _jsx(_components.span, { className: "token punctuation", children: "}" })] }), " ", _jsx(_components.span, { className: "token parameter", children: "radians" }), " \n"] }) }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token doc-comment comment", children: " */" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method-variable function-variable method function property-access", children: "degrees" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token parameter", children: "radians" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token arrow operator", children: "=>" }), " radians ", _jsx(_components.span, { className: "token operator", children: "*" }), " ", _jsx(_components.span, { className: "token number", children: "180" }), " ", _jsx(_components.span, { className: "token operator", children: "/" }), " ", _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token constant", children: "PI" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] })] }) }), "\n", _jsx(_components.h3, { children: "Bucket.js" }), "\n", _jsx(_components.pre, { className: "language-javascript", children: _jsxs(_components.code, { className: "language-javascript code-highlight", children: [_jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token keyword", children: "class" }), " ", _jsx(_components.span, { className: "token class-name", children: "Bucket" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "constructor" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsxs(_components.span, { className: "token parameter", children: ["bucketCount", _jsx(_components.span, { className: "token punctuation", children: "," }), " rangeLow", _jsx(_components.span, { className: "token punctuation", children: "," }), " rangeHigh"] }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "bucketCount" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " bucketCount", _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "rangeLow" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " rangeLow", _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "rangeHigh" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " rangeHigh", _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "stepSize" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "abs" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "rangeLow", _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token operator", children: "+" }), " ", _jsx(_components.span, { className: "token known-class-name class-name", children: "Math" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "abs" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "rangeHigh", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token operator", children: "/" }), " bucketCount", _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "buckets" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// For easiness we include the indexes" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "bucketIndexes" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "for" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "let" }), " i ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "0" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " i ", _jsx(_components.span, { className: "token operator", children: "<" }), " bucketCount", _jsx(_components.span, { className: "token punctuation", children: ";" }), " i", _jsx(_components.span, { className: "token operator", children: "++" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "bucketIndexes" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "push" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "i", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "initBuckets" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "initBuckets" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "for" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token keyword", children: "let" }), " i ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "0" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " i ", _jsx(_components.span, { className: "token operator", children: "<" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "bucketCount" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), " i", _jsx(_components.span, { className: "token operator", children: "++" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "buckets" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "i", _jsx(_components.span, { className: "token punctuation", children: "]" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token punctuation", children: "[" }), _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "addValueToBucket" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token parameter", children: "val" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " idx ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "getBucketIdxForValue" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "val", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "buckets" }), _jsx(_components.span, { className: "token punctuation", children: "[" }), "idx", _jsx(_components.span, { className: "token punctuation", children: "]" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token method function property-access", children: "push" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "val", _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token function", children: "getBucketIdxForValue" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token parameter", children: "val" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword", children: "let" }), " idx ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token number", children: "0" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token comment", children: "// Find bucket, put values on the outer size of the range in the last bucket" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "while" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), _jsx(_components.span, { className: "token punctuation", children: "(" }), "idx ", _jsx(_components.span, { className: "token operator", children: "<" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "bucketCount" }), " ", _jsx(_components.span, { className: "token operator", children: "-" }), " ", _jsx(_components.span, { className: "token number", children: "1" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token operator", children: "&&" }), " val ", _jsx(_components.span, { className: "token operator", children: ">" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "rangeLow" }), " ", _jsx(_components.span, { className: "token operator", children: "+" }), " ", _jsx(_components.span, { className: "token keyword", children: "this" }), _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "stepSize" }), " ", _jsx(_components.span, { className: "token operator", children: "*" }), " ", _jsx(_components.span, { className: "token punctuation", children: "(" }), "idx ", _jsx(_components.span, { className: "token operator", children: "+" }), " ", _jsx(_components.span, { className: "token number", children: "1" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), _jsx(_components.span, { className: "token punctuation", children: ")" }), " ", _jsx(_components.span, { className: "token punctuation", children: "{" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" idx", _jsx(_components.span, { className: "token operator", children: "++" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token keyword control-flow", children: "return" }), " idx", _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [" ", _jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsxs(_components.span, { className: "code-line", children: [_jsx(_components.span, { className: "token punctuation", children: "}" }), "\n"] }), _jsx(_components.span, { className: "code-line", children: "\n" }), _jsxs(_components.span, { className: "code-line", children: ["module", _jsx(_components.span, { className: "token punctuation", children: "." }), _jsx(_components.span, { className: "token property-access", children: "exports" }), " ", _jsx(_components.span, { className: "token operator", children: "=" }), " ", _jsx(_components.span, { className: "token maybe-class-name", children: "Bucket" }), _jsx(_components.span, { className: "token punctuation", children: ";" }), "\n"] })] }) }), "\n", _jsx(_components.h2, { children: "References" }), "\n", _jsx(_components.p, { children: "While creating this solution, I used the following articles as a reference." }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "https://medium.com/@tuzzer/cart-pole-balancing-with-q-learning-b54c6068d947", children: "https://medium.com/@tuzzer/cart-pole-balancing-with-q-learning-b54c6068d947" }) }), "\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "https://ferdinand-muetsch.de/cartpole-with-qlearning-first-experiences-with-openai-gym.html", children: "https://ferdinand-muetsch.de/cartpole-with-qlearning-first-experiences-with-openai-gym.html" }) }), "\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "https://threads-iiith.quora.com/Deep-Q-Learning-with-Neural-Networks-on-Cart-Pole", children: "https://threads-iiith.quora.com/Deep-Q-Learning-with-Neural-Networks-on-Cart-Pole" }) }), "\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "https://gist.github.com/n1try/af0b8476ae4106ec098fea1dfe57f578", children: "https://gist.github.com/n1try/af0b8476ae4106ec098fea1dfe57f578" }) }), "\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "https://gist.github.com/n1try/2a6722407117e4d668921fce53845432#file-dqn_cartpole-py", children: "https://gist.github.com/n1try/2a6722407117e4d668921fce53845432#file-dqn_cartpole-py" }) }), "\n"] })] }); } } return { default: MDXContent };

Are you struggling with testing and integrating AI into your product?

Are you struggling with testing and integrating AI into your product?