Separating Training And Testing Sets

long cutoffTimestamp = getTimestamp("2024-04-15T00:00:00");
// . . .
Candle[] trainingCandles = this._allCandles[c.BTCUSDT]
				.filter!(c => c.timestamp < cutoffTimestamp)
				.array;

// predictions from 2024-04-15 to 2024-04-28 (inclusive)
if (isSkippingPredictions) {
	if (this.candles[c.BTCUSDT][$-1].timestamp >= cutoffTimestamp) {
		isSkippingPredictions = false;
	}
	return true;
}

module bbot.bot012;

import std.stdio : pp = writeln;
import std.algorithm : map, sum, filter;
import std.algorithm.comparison : min;
import std.array : array;
import std.algorithm.searching : minElement;
import std.range : repeat, take;
import std.math.trigonometry : tanh;
import std.conv : to;
import std.random : Random, choice, uniform01, uniform;
import std.range : iota, repeat, generate, take;

import c = blib.constants;
import bbot.badgerbot : AbstractBadger;
import bmodel.chart : Chart;
import bmodel.candle : Candle;
import bmodel.order : Order;
import bmodel.enums : Interval, OrderType, Direction, Action, Color;

import blib.utils : getTimestamp;


class BadgerBot : AbstractBadger {
	override void setup() {
		this.TRADE_LIVE = false; // ATTENTION !!!

		// ignored when running LIVE bot (or pulling real values)
		this.capital = 100_000;
		this.transactionFee = 0.001; // 0.1% -> approx like this on Binance
		this.backtestPeriod = ["2024-04-01T00:00:00", "2024-04-29T00:00:00"]; // semi-open interval: [>
		// backtestPeriod ignored on LIVE - calculated from pastCandlesCount, candleInterval (and current timestamp)

		// applied always
		this.candleInterval = Interval._1M;
		this.pastCandlesCount = 4; // example: set to 50 for 50-candle moving average (limit 1000 candles total for LIVE - for simplicity)
		this.tickers = [c.BTCUSDT]; // ETHUSDT
		//this.chart = Chart(450, 200); // comment out when you want to visualize like on LIVE but backtest will (intentionally) run slower

		// this.positions; // will be autofilled
		// this.candles; // will be autofilled
		super.setup(); // DON'T REMOVE
	}

	// TARGET/STOP PRICE PREDICTIONS

	bool hasOpenPosition() {
		return 
			this.positions.length != 0 &&
			this.positions[$-1].action == Action.OPEN;
	}

	bool isAIrained = false;
	bool isSkippingPredictions = true;
	auto rand = Random(1);

	double[] backupWeights1 = [];
	double[] backupWeights2 = [];
	double[] backupWeights3 = [];

	// 1st neuron, 1st layer
	double[] weights1Layer1 = repeat(0.5, 10).array; // 9+1
	// 2nd neuron, 1st layer
	double[] weights2Layer1 = repeat(0.5, 10).array; // 9+1
	// 1st neuron, 2nd layer
	double[] weights1Layer2 = repeat(0.5, 3).array;  // 2+1

	double sigmoid(double input) { return (tanh(input) + 1) / 2.0; }

	double[] getInputs(Candle c1, Candle c2, Candle c3) {
		double[] inputs1 = [c1.high, c1.low, c1.close];
		inputs1[] -= c1.open;
		inputs1[] /= c1.open;
		double[] inputs2 = [c2.high, c2.low, c2.close];
		inputs1[] -= c2.open;
		inputs1[] /= c2.open;
		double[] inputs3 = [c3.high, c3.low, c3.close];
		inputs1[] -= c3.open;
		inputs1[] /= c3.open;
		return inputs1 ~ inputs2 ~ inputs3;
	}

	double getCorrectOutput(int idx, double targetPrice, double stopPrice) {
		// return 1 if target hit
		// return 0 if stop loss hit
		// return -1 if nothing was reached (end of data)
		int returnValue = -1;
		foreach(c; _allCandles[c.BTCUSDT][(idx+1)..$]) {
			if(c.low <= stopPrice) {
				returnValue = 0;
				break;
			}
			if(c.high >= targetPrice) {
				returnValue = 1;
				break;
			}
		}
		return returnValue;
	}

	Color getPredictedColor(double prediction) {
		return (prediction > 0.5) ? Color.GREEN : Color.RED;
	}

	double getPrediction(double[] inputs, double[] weights) {
		double[] weightedInputs = [1.0] ~ inputs;
		weightedInputs[] *=  weights[];
		double weightedSum = weightedInputs.array.sum;
		double prediction = sigmoid(weightedSum);
		return prediction;
	}

	void backupWeights() {
		backupWeights1 = weights1Layer1.dup;
		backupWeights2 = weights2Layer1.dup;
		backupWeights3 = weights1Layer2.dup;
	}

	void restoreWeights() {
		weights1Layer1 = backupWeights1.dup;
		weights2Layer1 = backupWeights2.dup;
		weights1Layer2 = backupWeights3.dup;
	}

	void updateRandomWeights() { // [-3,+3]
		// square root of all weights (23) is ~ 4.8
		// so I'm gonna update sample of 4 each time
		backupWeights();
		for (int i = 0; i < 4; i++) {
			int idx = uniform(0, 23, rand);
			if (idx <= 9) {
				weights1Layer1[idx] = uniform01(rand)*6-3;
			}
			if (idx >= 10 && idx <= 19) {
				weights2Layer1[idx-10] = uniform01(rand)*6-3;
			}
			if (idx >= 20 && idx <= 22) {
				weights1Layer2[idx-20] = uniform01(rand)*6-3;
			}
		}
	}

	void trainNetwork(Candle[] trainingCandles) {
		double maxError = trainingCandles.length;
		for (int n = 0; n < 1000; n++) { // 1000 times through the data
			double allDataError = 0;
			string printing = "";
			updateRandomWeights();
			for (int i = 2; i < trainingCandles.length - 1; i++) {
				double[] inputs = getInputs(
							    trainingCandles[i],
							    trainingCandles[i-1],
							    trainingCandles[i-2]);
				double prediction1Layer1 = getPrediction(inputs, weights1Layer1);
				double prediction2Layer1 = getPrediction(inputs, weights2Layer1);
				double prediction1Layer2 = getPrediction([prediction1Layer1, prediction2Layer1], weights1Layer2);
				double output = getCorrectOutput(i,
												 trainingCandles[i].close * 1.003, // +0.3% target price
												 trainingCandles[i].close * 0.995);// -0.5% stop price
				double error = prediction1Layer2 - output;
				if (output == -1) error = 0; // end or close-to-end of data => skipping
				allDataError += error * error;
				//printing ~= to!string(inputs) ~ " : " ~ to!string(prediction1Layer1) ~ " --> " ~to!string(output) ~ "\n";
			}
			if (allDataError < maxError) {
				maxError = allDataError;
				pp();
				pp("ERROR: ", maxError, " --> n : ", n);
				pp(weights1Layer1);
				pp(weights2Layer1);
				pp(weights1Layer2);
				//pp(printing);
			}
			else {
				restoreWeights();
			}
		}
		pp();
		pp("final weights:");
		pp(weights1Layer1);
		pp(weights2Layer1);
		pp(weights1Layer2);
		pp();
		pp("TRAINING COMPLETE");
		pp();
	}


	override bool trade() {
		if(!super.trade()) return false;
		// ALL CODE HERE - BEGIN //

		long cutoffTimestamp = getTimestamp("2024-04-15T00:00:00");

		if (!isAIrained) {
			// training on 2 weeks
			// from 2024-04-01 to 2024-04-14 (inclusive)
			// see ==> this.backtestPeriod
			// 2 weeks is plenty when trading minute data
			Candle[] trainingCandles = this._allCandles[c.BTCUSDT]
							.filter!(c => c.timestamp < cutoffTimestamp)
							.array;
			trainNetwork(trainingCandles);
			isAIrained = true;
		}

		// predictions from 2024-04-15 to 2024-04-28 (inclusive)
		if (isSkippingPredictions) {
			if (this.candles[c.BTCUSDT][$-1].timestamp >= cutoffTimestamp) {
				isSkippingPredictions = false;
			}
			return true;
		}

		Candle[] btcCandles = this.candles[c.BTCUSDT];
		double currentPrice = btcCandles[$-1].close;

		Candle lastCandle = btcCandles[$-2];
		Candle secondCandle = btcCandles[$-3];
		Candle thirdCandle = btcCandles[$-4];
		double[] inputs = getInputs(lastCandle, secondCandle, thirdCandle);
		double prediction1Layer1 = getPrediction(inputs, weights1Layer1);
		double prediction2Layer1 = getPrediction(inputs, weights2Layer1);
		double prediction1Layer2 = getPrediction([prediction1Layer1, prediction2Layer1], weights1Layer2);
		Color predictedNextCandleColor = getPredictedColor(prediction1Layer2);

		if (
			predictedNextCandleColor == Color.GREEN &&
			!hasOpenPosition()
		) {
			Order marketBuy = Order(
				c.BTCUSDT,
				OrderType.MARKET,
				Direction.BUY,
				currentPrice * 1.003, // target price = +0.3%
				currentPrice * 0.995, // stop price   = -0.5%
			);
			this.order(marketBuy);
		}

		// ALL CODE HERE - END //
		return true;
	}
}