function [T, R] = gridWorld

% Grid world layout:
%
%  ---------------------
%  |  1 |  2 |  3 |  4 |
%  ---------------------
%  |  5 |  6 |  7 |  8 |
%  ---------------------
%  |  9 | 10 | 11 | 12 |
%  ---------------------
%  | 13 | 14 | 15 | 16 |
%  ---------------------
%
%  Goal state: 16 
%  Bad state: 10
%  End state: 17
%
%  The end state is an absorbing state that the agent transitions 
%  to after visiting the goal state.
%
%  There are 17 states in total (including the end state) 
%  and 4 actions (up, down, right, left).
%

%%%%%%%%%%%%%%%%%%% rewards %%%%%%%%%%%%%%%%%%%%%%

% Rewards are stored in a one dimensional array R(s)
%
% All states have a reward of -1 except:
% Goal state: 100
% Bad state: -70
% End state: 0 

% initialize rewards to -1
R = -ones(17,1);

% set rewards
R(16) = 100;  % goal state
R(10) = -70;  % bad state
R(17) = 0;    % end state

%%%%%%%%%%%%%%%%%% transitions %%%%%%%%%%%%%%%%%%%

% Transitions are stored in a 3 dimensional array T(s,s',a) such that
% T(s,s',a) = Pr(s'|s,a)

% initialize all transition probabilities to 0
T = zeros(17,17,4);

% For a given action, the agent has probability 0.8 of moving 
% by one square in the intended direction and probability 0.1 
% of moving sideways.  When the agent bumps into a wall, 
% it stays in its current location.

% transition parameters
a = 0.8;  % intended move
b = 0.1;  % lateral move

% up (a = 1)

T(1,1,1) = a+b;
T(1,2,1) = b;

T(2,1,1) = b;
T(2,2,1) = a;
T(2,3,1) = b;

T(3,2,1) = b;
T(3,3,1) = a;
T(3,4,1) = b;

T(4,3,1) = b;
T(4,4,1) = a+b;

T(5,5,1) = b;
T(5,1,1) = a;
T(5,6,1) = b;

T(6,5,1) = b;
T(6,2,1) = a;
T(6,7,1) = b;

T(7,6,1) = b;
T(7,3,1) = a;
T(7,8,1) = b;

T(8,7,1) = b;
T(8,4,1) = a;
T(8,8,1) = b;

T(9,9,1) = b;
T(9,5,1) = a;
T(9,10,1) = b;

T(10,9,1) = b;
T(10,6,1) = a;
T(10,11,1) = b;

T(11,10,1) = b;
T(11,7,1) = a;
T(11,12,1) = b;

T(12,11,1) = b;
T(12,8,1) = a;
T(12,12,1) = b;

T(13,13,1) = b;
T(13,9,1) = a;
T(13,14,1) = b;

T(14,13,1) = b;
T(14,10,1) = a;
T(14,15,1) = b;

T(15,14,1) = b;
T(15,11,1) = a;
T(15,16,1) = b;

T(16,17,1) = 1;
T(17,17,1) = 1;

% down (a = 2)

T(1,1,2) = b;
T(1,5,2) = a;
T(1,2,2) = b;

T(2,1,2) = b;
T(2,6,2) = a;
T(2,3,2) = b;

T(3,2,2) = b;
T(3,7,2) = a;
T(3,4,2) = b;

T(4,3,2) = b;
T(4,8,2) = a;
T(4,4,2) = b;

T(5,5,2) = b;
T(5,9,2) = a;
T(5,6,2) = b;

T(6,5,2) = b;
T(6,10,2) = a;
T(6,7,2) = b;

T(7,6,2) = b;
T(7,11,2) = a;
T(7,8,2) = b;

T(8,7,2) = b;
T(8,12,2) = a;
T(8,8,2) = b;

T(9,9,2) = b;
T(9,13,2) = a;
T(9,10,2) = b;

T(10,9,2) = b;
T(10,14,2) = a;
T(10,11,2) = b;

T(11,10,2) = b;
T(11,15,2) = a;
T(11,12,2) = b;

T(12,11,2) = b;
T(12,16,2) = a;
T(12,12,2) = b;

T(13,13,2) = a+b;
T(13,14,2) = b;

T(14,13,2) = b;
T(14,14,2) = a;
T(14,15,2) = b;

T(15,14,2) = b;
T(15,15,2) = a;
T(15,16,2) = b;

T(16,17,2) = 1;
T(17,17,2) = 1;

% left (a = 3)

T(1,1,3) = a+b;
T(1,5,3) = b;

T(2,2,3) = b;
T(2,1,3) = a;
T(2,6,3) = b;

T(3,3,3) = b;
T(3,2,3) = a;
T(3,7,3) = b;

T(4,4,3) = b;
T(4,3,3) = a;
T(4,8,3) = b;

T(5,1,3) = b;
T(5,5,3) = a;
T(5,9,3) = b;

T(6,2,3) = b;
T(6,5,3) = a;
T(6,10,3) = b;

T(7,3,3) = b;
T(7,6,3) = a;
T(7,11,3) = b;

T(8,4,3) = b;
T(8,7,3) = a;
T(8,12,3) = b;

T(9,5,3) = b;
T(9,9,3) = a;
T(9,13,3) = b;

T(10,6,3) = b;
T(10,9,3) = a;
T(10,14,3) = b;

T(11,7,3) = b;
T(11,10,3) = a;
T(11,15,3) = b;

T(12,8,3) = b;
T(12,11,3) = a;
T(12,16,3) = b;

T(13,9,3) = b;
T(13,13,3) = a+b;

T(14,10,3) = b;
T(14,13,3) = a;
T(14,14,3) = b;

T(15,11,3) = b;
T(15,14,3) = a;
T(15,15,3) = b;

T(16,17,3) = 1;
T(17,17,3) = 1;

% right (a = 4)

T(1,1,4) = b;
T(1,2,4) = a;
T(1,5,4) = b;

T(2,2,4) = b;
T(2,3,4) = a;
T(2,6,4) = b;

T(3,3,4) = b;
T(3,4,4) = a;
T(3,7,4) = b;

T(4,4,4) = a+b;
T(4,8,4) = b;

T(5,1,4) = b;
T(5,6,4) = a;
T(5,9,4) = b;

T(6,2,4) = b;
T(6,7,4) = a;
T(6,10,4) = b;

T(7,3,4) = b;
T(7,8,4) = a;
T(7,11,4) = b;

T(8,4,4) = b;
T(8,8,4) = a;
T(8,12,4) = b;

T(9,5,4) = b;
T(9,10,4) = a;
T(9,13,4) = b;

T(10,6,4) = b;
T(10,11,4) = a;
T(10,14,4) = b;

T(11,7,4) = b;
T(11,12,4) = a;
T(11,15,4) = b;

T(12,8,4) = b;
T(12,12,4) = a;
T(12,16,4) = b;

T(13,9,4) = b;
T(13,14,4) = a;
T(13,13,4) = b;

T(14,10,4) = b;
T(14,15,4) = a;
T(14,14,4) = b;

T(15,11,4) = b;
T(15,16,4) = a;
T(15,15,4) = b;

T(16,17,4) = 1;
T(17,17,4) = 1;




