main.m

% MULTIMEDIA SYSTEMS | FEBRUARY 2024

videoPathRaw = "source.mp4";
videoPath = "converted_source.avi";

tic
    transformVideo(videoPathRaw);
display(newline +"TranformVideo runtime: "+toc);
tic
    calcErrorImageSequence(videoPath);
display(newline +"calcErrorImageSequence runtime: "+toc);
tic
    motionCompensation(videoPath);
display(newline +"motionCompensation runtime: "+toc);
tic
    deleteObject(videoPath);
display(newline +"deleteObject runtime: "+toc);

% topic 1 support function
% takes a standard colored video and converts it to 30fps
% uncompressed .avi. Also because of fps downscaling, it cuts the last
% 210frames from the original video, using my video, the result video is
% 5 seconds in length
function transformVideo(videoPath)
    v = VideoReader(videoPath);
    numOfFrames = v.NumFrames;
    tic
    original_sequence = read(v,[1,numOfFrames-210]); % this takes a lot of time
    display("Video read in: "+toc+"s.");

    % video save
    saveVideo(original_sequence,"converted_source.avi",30,'Uncompressed AVI')
end

% Topic 1.i)
function calcErrorImageSequence(videoPath)
    v = VideoReader(videoPath);
    numOfFrames = v.NumFrames;
    %numOfFrames =4; % debbug

    video_width = 640;
    video_height = 640;

    previousFrame = rgb2gray(readFrame(v)); % initialize the first frame as the first previous frame -> convert to grayscale
    previousFrame = imresize(im2uint8(previousFrame), [video_width,video_height]);

    error_img_sequence = zeros(video_width,video_height, numOfFrames - 1); % subtract the first I frame

    % for each frame starting from the second frame until end of frames
    for frame = 2:numOfFrames
        currentFrame = rgb2gray(readFrame(v)); % read next frame -> convert to grayscale -> set as current frame
        currentFrame = imresize(im2uint8(currentFrame), [video_width,video_height]);
        error_image = abs(double(currentFrame) - double(previousFrame)); % error image calculation
        error_img_sequence(:,:,frame-1) = error_image;

        previousFrame = currentFrame; % move to next frame
    end

    % figure window
    figure;
    for i = 1:3
        subplot(1, 3, i);
        imshow(error_img_sequence(:,:,i), []);
        title(['Error Image ' num2str(i)]);
    end
    sgtitle('Error Image Sequence [1 to 3]');

    % results, video player
    plyer = implay(uint8(error_img_sequence));
    set(plyer.Parent, 'Name', 'Error Image Sequence')

    % need to encode this sequnce with a lossless technique, i use
    % RLE (Run length encoding)

    % RLE encoder
    img_width = video_width ;
    img_height = video_height;
    compr_data = zeros(img_width*img_height*numOfFrames,2)+1;
    %comp_data = img_width*img_height*numOfFrames,2;
    index=1;
    for frame=1:numOfFrames-1
      for y =1:img_height
        for x=1:img_width          
            if(x == img_width)
                if(value == error_img_sequence(x,y,frame))
                    lengt = lengt + 1;
                    compr_data(index,:) = [lengt, value]; % store
                    index = index + 1;                   
                else
                    %store previous and reset and store 1
                    compr_data(index,:) = [lengt, value];
                    index = index + 1;

                    lengt = 1;
                    value = error_img_sequence(x,y,frame);
                    compr_data(index,:) = [lengt, value];
                    index = index + 1;
                end
            elseif (x == 1)
                %first x of each row
                value = error_img_sequence(x,y,frame);
                lengt = 1;
            else
                if ( error_img_sequence(x,y,frame) == value)
                    lengt = lengt + 1;
                else
                    % error_img_sequence(x,y,frame) ~= value
                    compr_data(index,:) = [lengt, value]; % store
                    index = index + 1;
                    %reset
                    value = error_img_sequence(x,y,frame);
                    lengt = 1;
                end
            end        
        end
      end
    end
    
    % RLE decoder
    % pixels_per_frame = img_width*img_height;
    pixels_per_row = img_width;

    decoded_sequence = zeros(video_width,video_height, numOfFrames - 1)+10; % instantiate 
    frame = 1; % current frame value
    pixel_n=1; % current pixel number (per frame)
    y=1; % current row value
    cc=0; % length loop step counter
    % display("Encoded sequence size: "+size(compr_data,1));
    for i=1:size(compr_data,1)
        for lengt=0:compr_data(i,1)-1
            if(y == (video_height+1))
                    frame=frame + 1;  
                    pixel_n = 1;
                    y = 1;
            end
            x = mod(pixel_n + lengt,pixels_per_row);
            if(x == 0) %640mod640=0
                x = pixels_per_row;
            end
            decoded_sequence( x, y ,frame) = compr_data(i,2);
            if(x == pixels_per_row) % after final x of row -> row+1
                y = y + 1;
            end
            cc = cc + 1;
        end
        pixel_n = pixel_n + cc;
        cc = 0;
    end

    % remove extra blank frames from decoded_sequence 
    decoded_length_with_blank_frames = size(decoded_sequence, 3);
    original_length = size(error_img_sequence,3);
    length_difference = decoded_length_with_blank_frames - original_length;
    decoded_sequence = decoded_sequence(:,:,1:end-length_difference);

    % display decoded_sequence
    plyer3 = implay(uint8(decoded_sequence));
    set(plyer3.Parent, 'Name', 'Decoded Image Sequence');

    % video save
    saveVideo(error_img_sequence,"error_video_file.avi",30,'Grayscale AVI')
end

% Topic 1.ii)
function motionCompensation(videoPath) 
    v = VideoReader(videoPath);

    % parameters
    macroblock_size = 64;
    video_width = 640;
    video_height = 640;

    % original_sequence = read(v,[1,131]); % debbug
    original_sequence = read(v); 
    
    resized_sequence = imresize(original_sequence,[video_width,video_height]); %Resize all frames (640,640)
    grayscale_sequence=zeros(video_width,video_height,size(resized_sequence,4));
    % convert all frames to grayscale
     for i=1:size(resized_sequence,4)
         grayscale_sequence(:,:,i) = rgb2gray(resized_sequence(:,:,:,i));
     end

    % encoded_sequence=zeros(video_width,video_height,size(grayscale_sequence,3));    
    encoded_sequence(:,:,1) = grayscale_sequence(:,:,1);

    % for all frames (while skipping the first) do:
    for iframe = 2:size(grayscale_sequence, 3)
        % create a macroblock meshgrid
        [X, Y] = meshgrid( 1:macroblock_size:video_width, 1:macroblock_size:video_height);

        % for all macroblocks
        for j = 1:numel(X)
            x = X(j);
            y = Y(j);

            currentMacroblock = grayscale_sequence( x: (x+macroblock_size-1) , y: (y+macroblock_size-1), iframe); % 64 x 64 macroblocks
            encoded_sequence(x:x+macroblock_size-1, y:y+macroblock_size-1, iframe) = ...
                blockmatching_search(currentMacroblock, x, y, grayscale_sequence(:,:,iframe-1)); % find & replace macroblock
        end
    end

    error_sequence = encoded_sequence - grayscale_sequence; % calc new error sequence

    decoded_sequence = encoded_sequence - error_sequence; % motion comp inversion

    % results
    pl0 = implay(uint8(error_sequence));
    set(pl0.Parent, 'Name', 'Error Sequence');
    pl1 = implay(uint8(encoded_sequence));
    set(pl1.Parent, 'Name', 'Motion Compensation Sequence');
    pl2 = implay(uint8(decoded_sequence));
    set(pl2.Parent, 'Name', 'Decoded Sequence');

    saveVideo(encoded_sequence,"motion_comp_video.avi",30,"Grayscale AVI");   
end

% topic 2
function deleteObject(videoPath)
    v = VideoReader(videoPath);

    % parameters
    macroblock_size = 16;
    video_width = 640;
    video_height = 640;

    % original_sequence = read(v,[120,131]);   % debugging
    original_sequence = read(v); 
    
    resized_sequence = imresize(original_sequence,[video_width,video_height]); %Resize all frames (640,640)
    grayscale_sequence=zeros(video_width,video_height,size(resized_sequence,4));

    % convert all frames to grayscale
     for i=1:size(resized_sequence,4)
         grayscale_sequence(:,:,i) = rgb2gray(resized_sequence(:,:,:,i));
     end

    encoded_sequence=zeros(video_width,video_height,size(grayscale_sequence,3));    
    encoded_sequence(:,:,1) = grayscale_sequence(:,:,1);
    
    % create a macroblock meshgrid
    [X, Y] = meshgrid( 1:macroblock_size:video_width, 1:macroblock_size:video_height);
    %display([X,Y]); % macroblock start cordinates

    % for all frames (while skipping the first) do:
    for iframe = 2:size(grayscale_sequence, 3)
        % for all macroblocks
        for j = 1:numel(X)
            x = X(j); % x - y are reversed x,y cordinates ( x -> y , y -> x )
            y = Y(j);
            
            % old 1 to 1 denied macroblocks
            %if( (x==257&&y==257) | (x==257&&y==257) | (x==257&&y==193) | (x==257&&y==257+64) | (x==257+64 && y==257+64) | (x==257+64 && y==257))
            
            % removed object zone
            if( ((x>=257 && y >= 225-16) && (x <= (257 + 16 + 16)) && y <= (225)) |((x>=257 && y >= 225) && (x <= (257 + 64)) && y <= (225+64)) | ((x>=257 && y >= 225+64) && (x <= (257 + 64 + 32) && y <= (225 + 64+ 32)) )) % 3 denial zones cause 3 different heights
                % entering this block will pass the macroblock denial area to the
                % blockmatching algorithm
                denied_x_range_start = x;
                denied_x_range_end = (x+macroblock_size-1);
                denied_y_range_start = y;
                denied_y_range_end = (y+macroblock_size-1);
                currentMacroblock = grayscale_sequence( x: (x+macroblock_size-1) , y: (y+macroblock_size-1), iframe); % 64 x 64 macroblocks
                encoded_sequence(x:x+macroblock_size-1, y:y+macroblock_size-1, iframe) = selective_blockmatching(currentMacroblock, x, y, grayscale_sequence(:,:,iframe-1),denied_x_range_start,denied_x_range_end,denied_y_range_start,denied_y_range_end,macroblock_size); % find & replace macroblock
            else
                % if out of denial zone, do not use motion compensation at
                % all
                encoded_sequence(x:x+macroblock_size-1, y:y+macroblock_size-1, iframe) = grayscale_sequence(x:x+macroblock_size-1, y:y+macroblock_size-1,iframe-1);
            end        
        end
    end

    encoded_sequence = encoded_sequence(:,:,2:end); % first frame has not been proceessed 

    % output
    pl1 = implay(uint8(encoded_sequence));
    set(pl1.Parent, 'Name', 'Motion Compensated Sequence');
    
    saveVideo(encoded_sequence,"object_removed_video.avi",30,"Grayscale AVI");
end

% takes an image sequence, saves it as video, converts image sequence to uint8 itself
function saveVideo(img_sequence,vidName, vidFrameRate, vidFormat)
    videoWriter = VideoWriter(vidName, vidFormat);
    videoWriter.FrameRate = vidFrameRate;
    open(videoWriter);
    writeVideo(videoWriter, uint8(img_sequence));
    close(videoWriter);
    fprintf("Video saved at pwd: "+ vidName);
end