Duplicate from pyesonekyaw/faceforgerydetection

Co-authored-by: Pye Sone Kyaw <[email protected]>

.gitattributes

@@ -0,0 +1,39 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+Weights/94_0.9485_val.tar filter=lfs diff=lfs merge=lfs -text
+Weights/FFc23.tar filter=lfs diff=lfs merge=lfs -text
+Examples/Fake1.mp4 filter=lfs diff=lfs merge=lfs -text
+Examples/Real1.mp4 filter=lfs diff=lfs merge=lfs -text
+*.dat filter=lfs diff=lfs merge=lfs -text

Examples/Fake1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57f42fe00febf36d3c5afb8a6caa7a7969b6fb6ec68799c5980cd994d611e28d
+size 5237358

Examples/Real1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3aa4657ca2627ed201577c61071d01729d9731c7fdcd8963eee816bb2a3420b4
+size 2090471

Examples/ReferenceVideos.txt

@@ -0,0 +1,5 @@
+Trying To Create an AI Tom Scott (on a $100 budget) - https://www.youtube.com/watch?v=xCn03u_Jvuo
+It’s Getting Harder to Spot a Deep Fake Video - https://www.youtube.com/watch?v=gLoI9hAX9dw
+Robot Linus Reviews a Keyboard - Deepfake - Cleave Truly Ergonomic - https://www.youtube.com/watch?v=34AmKPJNfCg
+Don't drop your internet! - https://www.youtube.com/watch?v=csnnfJi1j2g
+Bail Reform: Last Week Tonight with John Oliver (HBO) - https://www.youtube.com/watch?v=xQLqIWbc9VM

README.md

@@ -0,0 +1,14 @@
+---
+title: Generalizable Face Forgery Detection with Self-Blended Consistency Learning
+emoji: 🐰
+colorFrom: gray
+colorTo: pink
+sdk: gradio
+sdk_version: 3.9
+app_file: app.py
+pinned: false
+license: mit
+duplicated_from: pyesonekyaw/faceforgerydetection
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

Scripts/DeepFakeMask.py

@@ -0,0 +1,149 @@
+import logging
+import cv2
+import numpy as np
+
+logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
+
+class Mask():
+    """ Parent class for masks
+        the output mask will be <mask_type>.mask
+        channels: 1, 3 or 4:
+                    1 - Returns a single channel mask
+                    3 - Returns a 3 channel mask
+                    4 - Returns the original image with the mask in the alpha channel """
+
+    def __init__(self, landmarks, face, channels=4):
+        logger.info("Initializing %s: (face_shape: %s, channels: %s, landmarks: %s)",
+                     self.__class__.__name__, face.shape, channels, landmarks)
+        self.landmarks = landmarks
+        self.face = face
+        self.channels = channels
+
+        mask = self.build_mask()
+        self.mask = self.merge_mask(mask)
+        logger.info("Initialized %s", self.__class__.__name__)
+
+    def build_mask(self):
+        """ Override to build the mask """
+        raise NotImplementedError
+
+    def merge_mask(self, mask):
+        """ Return the mask in requested shape """
+        logger.info("mask_shape: %s", mask.shape)
+        assert self.channels in (1, 3, 4), "Channels should be 1, 3 or 4"
+        assert mask.shape[2] == 1 and mask.ndim == 3, "Input mask be 3 dimensions with 1 channel"
+
+        if self.channels == 3:
+            retval = np.tile(mask, 3)
+        elif self.channels == 4:
+            retval = np.concatenate((self.face, mask), -1)
+        else:
+            retval = mask
+
+        logger.info("Final mask shape: %s", retval.shape)
+        return retval
+
+
+class dfl_full(Mask):  # pylint: disable=invalid-name
+    """ DFL facial mask """
+    def build_mask(self):
+        mask = np.zeros(self.face.shape[0:2] + (1, ), dtype=np.float32)
+
+        nose_ridge = (self.landmarks[27:31], self.landmarks[33:34])
+        jaw = (self.landmarks[0:17],
+               self.landmarks[48:68],
+               self.landmarks[0:1],
+               self.landmarks[8:9],
+               self.landmarks[16:17])
+        eyes = (self.landmarks[17:27],
+                self.landmarks[0:1],
+                self.landmarks[27:28],
+                self.landmarks[16:17],
+                self.landmarks[33:34])
+        parts = [jaw, nose_ridge, eyes]
+
+        for item in parts:
+            merged = np.concatenate(item)
+            cv2.fillConvexPoly(mask, cv2.convexHull(merged), 255.)  # pylint: disable=no-member
+        return mask
+
+
+class components(Mask):  # pylint: disable=invalid-name
+    """ Component model mask """
+    def build_mask(self):
+        mask = np.zeros(self.face.shape[0:2] + (1, ), dtype=np.float32)
+
+        r_jaw = (self.landmarks[0:9], self.landmarks[17:18])
+        l_jaw = (self.landmarks[8:17], self.landmarks[26:27])
+        r_cheek = (self.landmarks[17:20], self.landmarks[8:9])
+        l_cheek = (self.landmarks[24:27], self.landmarks[8:9])
+        nose_ridge = (self.landmarks[19:25], self.landmarks[8:9],)
+        r_eye = (self.landmarks[17:22],
+                 self.landmarks[27:28],
+                 self.landmarks[31:36],
+                 self.landmarks[8:9])
+        l_eye = (self.landmarks[22:27],
+                 self.landmarks[27:28],
+                 self.landmarks[31:36],
+                 self.landmarks[8:9])
+        nose = (self.landmarks[27:31], self.landmarks[31:36])
+        parts = [r_jaw, l_jaw, r_cheek, l_cheek, nose_ridge, r_eye, l_eye, nose]
+
+        for item in parts:
+            merged = np.concatenate(item)
+            cv2.fillConvexPoly(mask, cv2.convexHull(merged), 255.)  # pylint: disable=no-member
+        return mask
+
+
+class extended(Mask):  # pylint: disable=invalid-name
+    """ Extended mask
+        Based on components mask. Attempts to extend the eyebrow points up the forehead
+    """
+    def build_mask(self):
+        mask = np.zeros(self.face.shape[0:2] + (1, ), dtype=np.float32)
+
+        landmarks = self.landmarks.copy()
+        # mid points between the side of face and eye point
+        ml_pnt = (landmarks[36] + landmarks[0]) // 2
+        mr_pnt = (landmarks[16] + landmarks[45]) // 2
+
+        # mid points between the mid points and eye
+        ql_pnt = (landmarks[36] + ml_pnt) // 2
+        qr_pnt = (landmarks[45] + mr_pnt) // 2
+
+        # Top of the eye arrays
+        bot_l = np.array((ql_pnt, landmarks[36], landmarks[37], landmarks[38], landmarks[39]))
+        bot_r = np.array((landmarks[42], landmarks[43], landmarks[44], landmarks[45], qr_pnt))
+
+        # Eyebrow arrays
+        top_l = landmarks[17:22]
+        top_r = landmarks[22:27]
+
+        # Adjust eyebrow arrays
+        landmarks[17:22] = top_l + ((top_l - bot_l) // 2)
+        landmarks[22:27] = top_r + ((top_r - bot_r) // 2)
+
+        r_jaw = (landmarks[0:9], landmarks[17:18])
+        l_jaw = (landmarks[8:17], landmarks[26:27])
+        r_cheek = (landmarks[17:20], landmarks[8:9])
+        l_cheek = (landmarks[24:27], landmarks[8:9])
+        nose_ridge = (landmarks[19:25], landmarks[8:9],)
+        r_eye = (landmarks[17:22], landmarks[27:28], landmarks[31:36], landmarks[8:9])
+        l_eye = (landmarks[22:27], landmarks[27:28], landmarks[31:36], landmarks[8:9])
+        nose = (landmarks[27:31], landmarks[31:36])
+        parts = [r_jaw, l_jaw, r_cheek, l_cheek, nose_ridge, r_eye, l_eye, nose]
+
+        for item in parts:
+            merged = np.concatenate(item)
+            cv2.fillConvexPoly(mask, cv2.convexHull(merged), 255.)  # pylint: disable=no-member
+        return mask
+
+
+class facehull(Mask):  # pylint: disable=invalid-name
+    """ Basic face hull mask """
+    def build_mask(self):
+        mask = np.zeros(self.face.shape[0:2] + (1, ), dtype=np.float32)
+        hull = cv2.convexHull(  # pylint: disable=no-member
+            np.array(self.landmarks).reshape((-1, 2)))
+        cv2.fillConvexPoly(mask, hull, 255.0, lineType=cv2.LINE_AA)  # pylint: disable=no-member
+        return mask

Scripts/ca_generator.py

@@ -0,0 +1,32 @@
+import albumentations as alb
+from albumentations.pytorch import ToTensorV2
+import cv2
+
+def get_augs(name):
+    IMG_SIZE = 380  
+    if name == "REAlbu":
+        return alb.Compose([
+            alb.HorizontalFlip(),
+            alb.CoarseDropout(max_holes = 1, min_height=int(IMG_SIZE*0.02), max_height=int(IMG_SIZE*0.2), min_width=int(IMG_SIZE*0.02), max_width=int(IMG_SIZE*0.2), p=1),
+        ])
+    elif name == "RandCropAlbu":
+        return alb.Compose([
+            alb.HorizontalFlip(),
+            alb.RandomResizedCrop(height = IMG_SIZE, width = IMG_SIZE, scale=(1/1.3, 1.0), ratio=(0.9,1.1)),
+        ])
+    elif name == "DFDCAlbu":
+        return alb.Compose([
+            alb.ImageCompression(quality_lower=60, quality_upper=100, p=0.5),
+            alb.GaussNoise(p=0.1),
+            alb.GaussianBlur(blur_limit=3, p=0.05),
+            alb.HorizontalFlip(),
+            alb.OneOf([
+                alb.LongestMaxSize(max_size=IMG_SIZE, interpolation=cv2.INTER_CUBIC),
+                alb.LongestMaxSize(max_size=IMG_SIZE, interpolation=cv2.INTER_AREA),
+                alb.LongestMaxSize(max_size=IMG_SIZE, interpolation=cv2.INTER_LINEAR)
+            ], p=1.0),
+            alb.PadIfNeeded(min_height=IMG_SIZE, min_width=IMG_SIZE, border_mode=cv2.BORDER_CONSTANT),
+            alb.OneOf([alb.RandomBrightnessContrast(), alb.FancyPCA(), alb.HueSaturationValue()], p=0.7),
+            alb.ToGray(p=0.2),
+            alb.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.2, rotate_limit=10, border_mode=cv2.BORDER_CONSTANT, p=0.5),
+        ])

Scripts/model.py

@@ -0,0 +1,34 @@
+import torch
+from torch import nn
+from efficientnet_pytorch import EfficientNet
+from pytorch_grad_cam import GradCAMElementWise
+from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget
+
+
+class Detector(nn.Module):
+    def __init__(self):
+        super(Detector, self).__init__()
+        self.net=EfficientNet.from_pretrained("efficientnet-b4",advprop=True,num_classes=2)
+
+    def forward(self,x):
+        x=self.net(x)
+        return x
+    
+
+def create_model(path="Weights/94_0.9485_val.tar", device=torch.device('cpu')):
+    model=Detector()
+    model=model.to(device)
+    if device == torch.device('cpu'):
+        cnn_sd=torch.load(path, map_location=torch.device('cpu') )["model"]
+    else:
+        cnn_sd=torch.load(path)["model"]
+    model.load_state_dict(cnn_sd)
+    model.eval()
+    return model
+
+def create_cam(model):
+    target_layers = [model.net._blocks[-1]]
+    targets = [ClassifierOutputTarget(1)]
+    cam_algorithm = GradCAMElementWise
+    cam = cam_algorithm(model=model,target_layers=target_layers,use_cuda=False)
+    return cam

Scripts/preprocess.py

@@ -0,0 +1,149 @@
+import numpy as np
+import cv2
+from tqdm import tqdm
+
+def extract_frames(filename,num_frames,model,image_size=(380,380)):
+	cap_org = cv2.VideoCapture(filename)
+	
+	if not cap_org.isOpened():
+		print(f'Cannot open: {filename}')
+		# sys.exit()
+		return []
+	
+	croppedfaces=[]
+	idx_list=[]
+	frame_count_org = int(cap_org.get(cv2.CAP_PROP_FRAME_COUNT))
+	
+	frame_idxs = np.linspace(0, frame_count_org - 1, num_frames, endpoint=True, dtype=int)
+	for cnt_frame in range(frame_count_org): 
+		ret_org, frame_org = cap_org.read()
+		height,width=frame_org.shape[:-1]
+		if not ret_org:
+			tqdm.write('Frame read {} Error! : {}'.format(cnt_frame,os.path.basename(filename)))
+			break
+		
+		if cnt_frame not in frame_idxs:
+			continue
+
+		frame = cv2.cvtColor(frame_org, cv2.COLOR_BGR2RGB)
+		
+		faces = model.predict_jsons(frame)
+		try:
+			if len(faces)==0:
+				tqdm.write('No faces in {}:{}'.format(cnt_frame,os.path.basename(filename)))
+				continue
+
+			size_list=[]
+			croppedfaces_temp=[]
+			idx_list_temp=[]
+			
+			for face_idx in range(len(faces)):
+				x0,y0,x1,y1=faces[face_idx]['bbox']
+				bbox=np.array([[x0,y0],[x1,y1]])
+				croppedfaces_temp.append(cv2.resize(crop_face(frame,None,bbox,False,crop_by_bbox=True,only_img=True,phase='test'),dsize=image_size).transpose((2,0,1)))
+				idx_list_temp.append(cnt_frame)
+				size_list.append((x1-x0)*(y1-y0))
+			
+			max_size=max(size_list)
+			croppedfaces_temp=[f for face_idx,f in enumerate(croppedfaces_temp) if size_list[face_idx]>=max_size/2]
+			idx_list_temp=[f for face_idx,f in enumerate(idx_list_temp) if size_list[face_idx]>=max_size/2]
+			croppedfaces+=croppedfaces_temp
+			idx_list+=idx_list_temp	
+		except Exception as e:
+			print(f'error in {cnt_frame}:{filename}')
+			print(e)
+			continue
+	cap_org.release()
+
+	
+
+	return croppedfaces,idx_list
+
+def extract_face(frame,model,image_size=(380,380)):
+	
+	
+	faces = model.predict_jsons(frame)
+
+	if len(faces[0]['bbox'])==0:
+		return []
+	croppedfaces=[]
+	for face_idx in range(len(faces)):
+		x0,y0,x1,y1=faces[face_idx]['bbox']
+		bbox=np.array([[x0,y0],[x1,y1]])
+		croppedfaces.append(cv2.resize(crop_face(frame,None,bbox,False,crop_by_bbox=True,only_img=True,phase='test'),dsize=image_size).transpose((2,0,1)))
+	
+	return croppedfaces
+
+
+def crop_face(img,landmark=None,bbox=None,margin=False,crop_by_bbox=True,abs_coord=False,only_img=False,phase='train'):
+	assert phase in ['train','val','test']
+
+	#crop face------------------------------------------
+	H,W=len(img),len(img[0])
+
+	assert landmark is not None or bbox is not None
+
+	H,W=len(img),len(img[0])
+	
+	if crop_by_bbox:
+		x0,y0=bbox[0]
+		x1,y1=bbox[1]
+		w=x1-x0
+		h=y1-y0
+		w0_margin=w/4#0#np.random.rand()*(w/8)
+		w1_margin=w/4
+		h0_margin=h/4#0#np.random.rand()*(h/5)
+		h1_margin=h/4
+	else:
+		x0,y0=landmark[:68,0].min(),landmark[:68,1].min()
+		x1,y1=landmark[:68,0].max(),landmark[:68,1].max()
+		w=x1-x0
+		h=y1-y0
+		w0_margin=w/8#0#np.random.rand()*(w/8)
+		w1_margin=w/8
+		h0_margin=h/2#0#np.random.rand()*(h/5)
+		h1_margin=h/5
+
+	
+
+	if margin:
+		w0_margin*=4
+		w1_margin*=4
+		h0_margin*=2
+		h1_margin*=2
+	elif phase=='train':
+		w0_margin*=(np.random.rand()*0.6+0.2)#np.random.rand()
+		w1_margin*=(np.random.rand()*0.6+0.2)#np.random.rand()
+		h0_margin*=(np.random.rand()*0.6+0.2)#np.random.rand()
+		h1_margin*=(np.random.rand()*0.6+0.2)#np.random.rand()	
+	else:
+		w0_margin*=0.5
+		w1_margin*=0.5
+		h0_margin*=0.5
+		h1_margin*=0.5
+			
+	y0_new=max(0,int(y0-h0_margin))
+	y1_new=min(H,int(y1+h1_margin)+1)
+	x0_new=max(0,int(x0-w0_margin))
+	x1_new=min(W,int(x1+w1_margin)+1)
+	
+	img_cropped=img[y0_new:y1_new,x0_new:x1_new]
+	if landmark is not None:
+		landmark_cropped=np.zeros_like(landmark)
+		for i,(p,q) in enumerate(landmark):
+			landmark_cropped[i]=[p-x0_new,q-y0_new]
+	else:
+		landmark_cropped=None
+	if bbox is not None:
+		bbox_cropped=np.zeros_like(bbox)
+		for i,(p,q) in enumerate(bbox):
+			bbox_cropped[i]=[p-x0_new,q-y0_new]
+	else:
+		bbox_cropped=None
+
+	if only_img:
+		return img_cropped
+	if abs_coord:
+		return img_cropped,landmark_cropped,bbox_cropped,(y0-y0_new,x0-x0_new,y1_new-y1,x1_new-x1),y0_new,y1_new,x0_new,x1_new
+	else:
+		return img_cropped,landmark_cropped,bbox_cropped,(y0-y0_new,x0-x0_new,y1_new-y1,x1_new-x1)

Scripts/sbi_generator.py

@@ -0,0 +1,217 @@
+from imutils import face_utils
+import numpy as np
+import random
+import albumentations as alb
+from .DeepFakeMask import dfl_full, extended, components, facehull
+import cv2
+
+def IoUfrom2bboxes(boxA, boxB):
+    xA = max(boxA[0], boxB[0])
+    yA = max(boxA[1], boxB[1])
+    xB = min(boxA[2], boxB[2])
+    yB = min(boxA[3], boxB[3])
+    interArea = max(0, xB - xA + 1) * max(0, yB - yA + 1)
+    boxAArea = (boxA[2] - boxA[0] + 1) * (boxA[3] - boxA[1] + 1)
+    boxBArea = (boxB[2] - boxB[0] + 1) * (boxB[3] - boxB[1] + 1)
+    iou = interArea / float(boxAArea + boxBArea - interArea)
+    return iou
+
+def reorder_landmark(landmark):
+    landmark_add = np.zeros((13, 2))
+    for idx, idx_l in enumerate([77, 75, 76, 68, 69, 70, 71, 80, 72, 73, 79, 74, 78]):
+        landmark_add[idx] = landmark[idx_l]
+    landmark[68:] = landmark_add
+    return landmark
+
+def get_dlib_landmarks(inp, dlib_face_detector, dlib_face_predictor):
+    faces = dlib_face_detector(inp, 1)
+    if len(faces)==0:
+        raise Exception("No faces detected")
+    landmarks=[]
+    size_list=[]
+    for face_idx in range(len(faces)):
+        landmark = dlib_face_predictor(inp, faces[face_idx])
+        landmark = face_utils.shape_to_np(landmark)
+        x0,y0=landmark[:,0].min(),landmark[:,1].min()
+        x1,y1=landmark[:,0].max(),landmark[:,1].max()
+        face_s=(x1-x0)*(y1-y0)
+        size_list.append(face_s)
+        landmarks.append(landmark)
+    landmarks=np.concatenate(landmarks).reshape((len(size_list),)+landmark.shape)
+    landmarks=landmarks[np.argsort(np.array(size_list))[::-1]]
+    return landmarks
+
+def get_retina_bbox(inp,face_detector):
+    faces = face_detector.predict_jsons(inp)
+    landmarks=[]
+    size_list=[]
+    for face_idx in range(len(faces)):
+        
+        x0,y0,x1,y1=faces[face_idx]['bbox']
+        landmark=np.array([[x0,y0],[x1,y1]]+faces[face_idx]['landmarks'])
+        face_s=(x1-x0)*(y1-y0)
+        size_list.append(face_s)
+        landmarks.append(landmark)
+    landmarks=np.concatenate(landmarks).reshape((len(size_list),)+landmark.shape)
+    landmarks=landmarks[np.argsort(np.array(size_list))[::-1]]
+
+    return landmarks
+
+def random_get_hull(landmark,img, face_region):
+    face_region = int(face_region)
+    if face_region == 1:
+        mask = dfl_full(landmarks=landmark.astype('int32'),face=img, channels=3).mask
+    elif face_region == 2:
+        mask = extended(landmarks=landmark.astype('int32'),face=img, channels=3).mask
+    elif face_region == 3:
+        mask = components(landmarks=landmark.astype('int32'),face=img, channels=3).mask
+    else:
+        mask = facehull(landmarks=landmark.astype('int32'),face=img, channels=3).mask
+    return mask/255
+
+class RandomDownScale(alb.core.transforms_interface.ImageOnlyTransform):
+	def apply(self,img,**params):
+		return self.randomdownscale(img)
+
+	def randomdownscale(self,img):
+		keep_ratio=True
+		keep_input_shape=True
+		H,W,C=img.shape
+		ratio_list=[2,4]
+		r=ratio_list[np.random.randint(len(ratio_list))]
+		img_ds=cv2.resize(img,(int(W/r),int(H/r)),interpolation=cv2.INTER_NEAREST)
+		if keep_input_shape:
+			img_ds=cv2.resize(img_ds,(W,H),interpolation=cv2.INTER_LINEAR)
+
+		return img_ds
+
+def get_source_transforms():
+    return alb.Compose([
+        alb.Compose([
+            alb.RGBShift((-20, 20), (-20, 20), (-20, 20), p=0.3),
+            alb.HueSaturationValue(
+                hue_shift_limit=(-0.3, 0.3), sat_shift_limit=(-0.3, 0.3), val_shift_limit=(-0.3, 0.3), p=1),
+            alb.RandomBrightnessContrast(
+                brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=1),
+        ], p=1),
+
+        alb.OneOf([
+            RandomDownScale(p=1),
+            alb.Sharpen(alpha=(0.2, 0.5), lightness=(0.5, 1.0), p=1),
+        ], p=1),
+
+    ], p=1.)
+
+def randaffine(img, mask):
+    f = alb.Affine(
+        translate_percent={'x': (-0.03, 0.03), 'y': (-0.015, 0.015)},
+        scale=[0.95, 1/0.95],
+        fit_output=False,
+        p=1)
+
+    g = alb.ElasticTransform(
+        alpha=50,
+        sigma=7,
+        alpha_affine=0,
+        p=1,
+    )
+
+    transformed = f(image=img, mask=mask)
+    img = transformed['image']
+
+    mask = transformed['mask']
+    transformed = g(image=img, mask=mask)
+    mask = transformed['mask']
+    return img, mask
+
+def get_blend_mask(mask):
+	H,W=mask.shape
+	size_h=np.random.randint(192,257)
+	size_w=np.random.randint(192,257)
+	mask=cv2.resize(mask,(size_w,size_h))
+	kernel_1=random.randrange(5,26,2)
+	kernel_1=(kernel_1,kernel_1)
+	kernel_2=random.randrange(5,26,2)
+	kernel_2=(kernel_2,kernel_2)
+	
+	mask_blured = cv2.GaussianBlur(mask, kernel_1, 0)
+	mask_blured = mask_blured/(mask_blured.max())
+	mask_blured[mask_blured<1]=0
+	
+	mask_blured = cv2.GaussianBlur(mask_blured, kernel_2, np.random.randint(5,46))
+	mask_blured = mask_blured/(mask_blured.max())
+	mask_blured = cv2.resize(mask_blured,(W,H))
+	return mask_blured.reshape((mask_blured.shape+(1,)))
+
+
+def dynamic_blend(source,target,mask,blending_type, mixup_ratio=[0.25,0.5,0.75,1,1,1]):
+    """Performs dynamic blending of source and target, using the mask as the blending region
+
+    Args:
+        source: source image
+        target: target image
+        mask: mask image
+
+    Returns:
+        img_blended: blended image
+        mask_blurred: augmented mask used for blending
+    """
+
+    mask_blured = get_blend_mask(mask)
+    mask_blured_copy = mask_blured.copy()
+    
+    if blending_type == "Poisson":
+        # Poisson blending
+        b_mask = (mask_blured_copy * 255).astype(np.uint8)
+        l, t, w, h = cv2.boundingRect(b_mask)
+        center = (int(l + w / 2), int(t + h / 2))
+        img_blended = cv2.seamlessClone(source, target, b_mask, center, cv2.NORMAL_CLONE)
+    else:
+        # Mix up blending
+        blend_list=mixup_ratio
+        blend_ratio = blend_list[np.random.randint(len(blend_list))]
+
+        mask_blured_copy = mask_blured.copy()
+        mask_blured_copy*=blend_ratio
+
+        img_blended=(mask_blured_copy * source + (1 - mask_blured_copy) * target)
+
+    return img_blended,mask_blured
+
+def get_transforms():
+    return alb.Compose([
+
+        alb.RGBShift((-20, 20), (-20, 20), (-20, 20), p=0.3),
+        alb.HueSaturationValue(
+            hue_shift_limit=(-0.3, 0.3), sat_shift_limit=(-0.3, 0.3), val_shift_limit=(-0.3, 0.3), p=0.3),
+        alb.RandomBrightnessContrast(
+            brightness_limit=(-0.3, 0.3), contrast_limit=(-0.3, 0.3), p=0.3),
+        alb.ImageCompression(quality_lower=40, quality_upper=100, p=0.5),
+
+    ],
+        additional_targets={f'image1': 'image'},
+        p=1.)
+
+
+def self_blending(img, landmark, blending_type, face_region):
+    if np.random.rand() < 0.25:
+        landmark = landmark[:68]
+    mask = random_get_hull(landmark, img, face_region)
+    if mask.shape[-1] == 3:
+        mask = mask[:, :, 0]
+
+    mask_copy = mask
+
+    source_transforms = get_source_transforms()
+    source = img.copy()
+    source = source_transforms(image=source.astype(np.uint8))['image']
+
+    source_before_affine_transforms, mask_before_affine_transforms = source, mask
+    source, mask = randaffine(source, mask)
+    source_after_affine_transforms, mask_after_affine_transforms = source, mask
+
+    img_blended, mask = dynamic_blend(source, img, mask, blending_type)
+    img_blended = img_blended.astype(np.uint8)
+    img = img.astype(np.uint8)
+
+    return img, img_blended, mask, mask_copy, source_before_affine_transforms, mask_before_affine_transforms, source_after_affine_transforms, mask_after_affine_transforms

Weights/94_0.9485_val.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c100e98694faa776fcf71990377f9c4eca46568417339c06843cdcf2a78d35d
+size 141291061

Weights/FFc23.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b05c2ba36ccb9e9f4f9e1aae9acd443ae2e6400ce725f56104fdb175d3c3267
+size 141290933

Weights/README.md

@@ -0,0 +1,13 @@
+---
+title: Faceforgerydetection
+emoji: 💩
+colorFrom: gray
+colorTo: pink
+sdk: gradio
+sdk_version: 3.9
+app_file: app.py
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

Weights/shape_predictor_81_face_landmarks.dat

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cae4375589dd915d9a0a881101bed1bbb4e9887e35e63b024388f1ca25ff869
+size 19743860

app.py

@@ -0,0 +1,387 @@
+import warnings
+import cv2
+import dlib
+from pytorch_grad_cam.utils.image import show_cam_on_image
+from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget
+import gradio as gr
+import numpy as np
+import torch
+from retinaface.pre_trained_models import get_model
+
+from Scripts.model import create_cam, create_model
+from Scripts.preprocess import crop_face, extract_face, extract_frames
+from Scripts.ca_generator import get_augs
+from Scripts.sbi_generator import (IoUfrom2bboxes, get_dlib_landmarks,
+                                   get_retina_bbox, get_transforms,
+                                   reorder_landmark, self_blending)
+
+warnings.filterwarnings('ignore')
+
+
+# Model Initialization
+device = torch.device('cpu')
+
+sbcl = create_model("Weights/94_0.9485_val.tar")
+sbi = create_model("Weights/FFc23.tar")
+
+# Face Detector Initialization
+face_detector = get_model("resnet50_2020-07-20", max_size=1024, device=device)
+face_detector.eval()
+
+# Grad-CAM Initialization
+cam_sbi = create_cam(sbi)
+cam_sbcl = create_cam(sbcl)
+targets = [ClassifierOutputTarget(1)]
+
+# Examples
+examples = ["Examples/Fake/fake1.png", "Examples/Real/real1.png", "Examples/Real/real2.png", "Examples/Fake/fake3.png", "Examples/Real/real3.png",
+            "Examples/Fake/fake4.png", "Examples/Real/real4.png", "Examples/Fake/fake5.png", "Examples/Fake/fake6.png", "Examples/Fake/fake7.png", ]
+examples_videos = ['Examples/Fake1.mp4', 'Examples/Real1.mp4']
+examples_sbi = ["Examples/Fake/fake1.png", "Examples/Real/real1.png", "Examples/Real/real2.png", "Examples/Fake/fake3.png", "Examples/Real/real3.png",
+                "Examples/Fake/fake4.png", "Examples/Fake/fake5.png", ]
+
+# dlib Models
+dlib_face_detector = dlib.get_frontal_face_detector()
+dlib_face_predictor = dlib.shape_predictor(
+    'Weights/shape_predictor_81_face_landmarks.dat')
+
+
+def generate_sbi(inp, blending_type, face_region):
+    """
+    Visualizes the different steps in the self-blended image generation process for both RGB image and mask
+    """
+    # Getting face bboxes and landmarks
+    landmark = get_dlib_landmarks(
+        inp, dlib_face_detector, dlib_face_predictor)[0]
+    bbox_lm = np.array([landmark[:, 0].min(), landmark[:, 1].min(),
+                        landmark[:, 0].max(), landmark[:, 1].max()])
+    bboxes = get_retina_bbox(inp, face_detector)[:2]
+
+    # Reducing bboxes to just one if multiple
+    iou_max = -1
+    for i in range(len(bboxes)):
+        iou = IoUfrom2bboxes(bbox_lm, bboxes[i].flatten())
+        if iou_max < iou:
+            bbox = bboxes[i]
+            iou_max = iou
+
+    # Input cropping
+    landmarks = reorder_landmark(landmark)
+    img, landmarks, bbox, __ = crop_face(
+        inp, landmarks, bbox, margin=True, crop_by_bbox=False)
+    cropped_input_face = img
+
+    # Blending
+    img_r_before_both_transforms, img_f_before_both_transforms, mask, mask_original, source_before_affine_transforms, _, source_after_affine_transforms, mask_after_affine_transforms = self_blending(
+        img.copy(), landmark.copy(), blending_type, face_region)
+
+    # Post-blending transforms
+    transformed = get_transforms()(image=img_f_before_both_transforms.astype(
+        'uint8'), image1=img_r_before_both_transforms.astype('uint8'))
+    img_f_after_both_transforms, img_r_after_both_transforms = transformed[
+        'image'], transformed['image1']
+
+    # Crop and resize the faces
+    img_f, _, __, ___, y0_new, y1_new, x0_new, x1_new = crop_face(
+        img_f_after_both_transforms, landmark, bbox, margin=False, crop_by_bbox=True, abs_coord=True, phase='train')
+    img_r = img_r_after_both_transforms[y0_new:y1_new, x0_new:x1_new]
+    img_f, img_r = cv2.resize(img_f, (380, 380), interpolation=cv2.INTER_LINEAR), cv2.resize(
+        img_r, (380, 380), interpolation=cv2.INTER_LINEAR)
+
+    # Mask operations
+    mask, mask_original = cv2.cvtColor(mask, cv2.COLOR_GRAY2RGB), cv2.cvtColor(
+        mask_original, cv2.COLOR_GRAY2RGB)
+    mask_after_affine_transforms = cv2.cvtColor(
+        mask_after_affine_transforms, cv2.COLOR_GRAY2RGB)
+    return cropped_input_face, img_r_before_both_transforms, img_f_before_both_transforms, img_r_after_both_transforms, img_f_after_both_transforms,\
+         img_r, img_f, mask, mask_original, source_before_affine_transforms, source_after_affine_transforms, mask_after_affine_transforms
+
+
+def generate_ca(inp):
+    """
+    Applies consistency augmentations to the given input face
+    """
+    try:
+        face = extract_face(inp, face_detector)[0].transpose(1, 2, 0)
+    except:
+        raise Exception("No faces detected")
+    randomErasing, randomCropping, dfdc = get_augs("REAlbu"), get_augs("RandCropAlbu"), get_augs("DFDCAlbu")
+    return face, randomErasing(image=face)['image'], randomCropping(image=face)['image'], dfdc(image=face)['image']
+
+
+def predict_image(inp, model):
+    """
+    Performs inference for a given input image and returns the prediction and CAM image.
+    """
+    face_list = extract_face(inp, face_detector)
+
+    if len(face_list) == 0:
+        return {'No face detected!': 1}, None, None
+
+    with torch.no_grad():
+        img = torch.tensor(face_list).to(device).float()/255
+
+        if model == "Self-Blended Images":
+            pred = sbi(img).softmax(1)[:, 1].cpu().data.numpy().tolist()[0]
+        else:
+            pred = sbcl(img).softmax(1)[:, 1].cpu().data.numpy().tolist()[0]
+
+        confidences = {'Real': 1-pred, 'Fake': pred}
+
+    if model == "Self-Blended Images":
+        grayscale_cam = cam_sbi(
+            input_tensor=img, targets=targets, aug_smooth=True)
+    else:
+        grayscale_cam = cam_sbcl(
+            input_tensor=img, targets=targets, aug_smooth=True)
+    grayscale_cam = grayscale_cam[0, :]
+    cam_image = show_cam_on_image(face_list[0].transpose(
+        1, 2, 0)/255, grayscale_cam, use_rgb=True)
+
+    return confidences, cam_image
+
+
+def predict_video(inp, model):
+    """
+    Performs inference for a given input video and returns the prediction and CAM image of the frame with the highest fake probability.
+    """
+    face_list, idx_list = extract_frames(inp, 10, face_detector)
+
+    with torch.no_grad():
+        img = torch.tensor(face_list).to(device).float()/255
+        if model == "Self-Blended Images":
+            pred = sbi(img).softmax(1)[:, 1]
+        else:
+            pred = sbcl(img).softmax(1)[:, 1]
+
+    pred_list = []
+    idx_img = -1
+    for i in range(len(pred)):
+        if idx_list[i] != idx_img:
+            pred_list.append([])
+            idx_img = idx_list[i]
+        pred_list[-1].append(pred[i].item())
+    pred_res = np.zeros(len(pred_list))
+    for i in range(len(pred_res)):
+        pred_res[i] = max(pred_list[i])
+    pred = pred_res.mean()
+
+    most_fake = np.argmax(pred_res)
+    if model == "Self-Blended Images":
+        grayscale_cam = cam_sbi(input_tensor=img[most_fake].unsqueeze(
+            0), targets=targets, aug_smooth=True)
+    else:
+        grayscale_cam = cam_sbcl(input_tensor=img[most_fake].unsqueeze(
+            0), targets=targets, aug_smooth=True)
+    grayscale_cam = grayscale_cam[0, :]
+    cam_image = show_cam_on_image(face_list[most_fake].transpose(
+        1, 2, 0)/255, grayscale_cam, use_rgb=True)
+
+    return {'Real': 1-pred, 'Fake': pred}, cam_image
+
+
+with gr.Blocks(title="Self-Blended Consistency Learning", css="#custom_header {min-height: 3rem} #custom_title {min-height: 3rem; text-align: center}") as demo:
+    gr.Markdown("# Face Forgery Detector", elem_id="custom_title")
+    gr.Markdown("Gradio Demo for 'Face Forgery Detection with Self-Blended Consistency Learning'. To use it, simply upload your image, or click one of the examples to load them. Paper to be available on ArXiv in the near future.", elem_id="custom_title")
+
+    with gr.Tab("Image Inference"):
+        with gr.Row():
+            with gr.Column():
+                with gr.Box():
+                    gr.Markdown("## Inputs", elem_id="custom_header")
+                    input_image = gr.Image(label="Input Image")
+                    input_image.style(height=240)
+                    model_selection = gr.inputs.Radio(['Self-Blended Images', 'Self-Blended Consistency Learning'],
+                                                      type="value", default='Self-Blended Consistency Learning', label='Model')
+                    btn = gr.Button(value="Submit")
+                    btn.style(full_width=True)
+            with gr.Column():
+                with gr.Box():
+                    gr.Markdown("## Outputs", elem_id="custom_header")
+                    output_image = gr.Image(label="GradCAM Image")
+                    output_image.style(height=240)
+                    label_probs = gr.outputs.Label()
+        gr.Examples(
+            examples=examples,
+            inputs=input_image,
+            outputs=output_image,
+            fn=predict_image,
+            cache_examples=False,
+        )
+    with gr.Tab("Video Inference"):
+        with gr.Row():
+            with gr.Column():
+                with gr.Box():
+                    gr.Markdown("## Inputs", elem_id="custom_header")
+                    input_video = gr.Video(label="Input Video")
+                    input_video.style(height=240)
+                    model_selection_video = gr.inputs.Radio(
+                        ['Self-Blended Images', 'Self-Blended Consistency Learning'], type="value", default='Self-Blended Consistency Learning', label='Model')
+                    btn_video = gr.Button(value="Submit")
+                    btn_video.style(full_width=True)
+
+            with gr.Column():
+                with gr.Box():
+                    gr.Markdown("## Outputs", elem_id="custom_header")
+                    output_image_video = gr.Image(label="GradCAM Image")
+                    output_image_video.style(height=240)
+                    label_probs_video = gr.outputs.Label()
+        gr.Examples(
+            examples=examples_videos,
+            inputs=input_video,
+            outputs=output_image_video,
+            fn=predict_video,
+            cache_examples=False,
+        )
+
+    with gr.Tab("SBI Generator"):
+        gr.Markdown("Input an image with a face to visualize the steps involved in the self-blended image (SBI) generation. Values for augmentations are randomly chosen. Blending type and face region can be varied. \
+                    This process is a slightly modified version of the process from 'Detecting Deepfakes with Self-Blended Images (CVPR 2022)'", elem_id="custom_header")
+        with gr.Row():
+            with gr.Column():
+                with gr.Box():
+                    gr.Markdown("## Inputs", elem_id="custom_header")
+                    input_image_sbi = gr.Image(label="Input Image")
+                    input_image_sbi.style(height=240)
+                    btn_sbi = gr.Button(value="Submit")
+                    btn_sbi.style(full_width=True)
+                    with gr.Row():
+                        blending_type = gr.Radio(
+                            ["Poisson", "Mixup"], label="Blending Type", value="Poisson", interactive=True)
+                        face_region = gr.Radio(
+                            ["1", "2", "3", "4"], label="Face Region", value="1", interactive=True)
+                    gr.Examples(
+                        examples=examples_sbi,
+                        inputs=input_image_sbi,
+                        fn=generate_sbi,
+                        cache_examples=False,
+                    )
+        with gr.Row():
+            with gr.Box():
+                with gr.Column():
+                    gr.Markdown("# Self-Blended Image Generation",
+                                elem_id="custom_header")
+
+                    with gr.Box():
+                        gr.Markdown("## Step 1", elem_id="custom_header")
+                        gr.Markdown(
+                            "Using facial landmarks models, obtain face bounding box and facial landmarks to crop face and produce mask.", elem_id="custom_header")
+                        with gr.Row():
+                            cropped_input_face = gr.Image(
+                                label="Input face after cropping")
+                            cropped_input_face.style(height=240)
+                            mask_original = gr.Image(label="Original mask")
+                            mask_original.style(height=240)
+                        gr.Markdown("The cropped input face is duplicated to become a 'source' face and a 'target' face. Eventually, the source face will be blended onto the target face after augmentations done below.", elem_id="custom_header")
+
+                    with gr.Box():
+                        gr.Markdown("## Step 2", elem_id="custom_header")
+                        gr.Markdown("Apply source-target augmentations",
+                                    elem_id="custom_header")
+                        with gr.Row():
+                            source_before_affine_transforms = gr.Image(
+                                label="Source face after source-target augmentations")
+                            source_before_affine_transforms.style(height=240)
+                        gr.Markdown("In this case, the source-target augmentations are applied to the source image for straight-forward visualization. In actual training,\
+                                     the augmentations are applied to either source or target face with 1:1 probability. Augmentations applied here \
+                                     include RGBShift, HueSaturationValue, RandomBrightnessContrast, RandomDownScale, Sharpen from Albumentations.")
+
+                    with gr.Box():
+                        gr.Markdown("## Step 3", elem_id="custom_header")
+                        gr.Markdown(
+                            "Apply affine/elastic augmentations to augmented source image/mask", elem_id="custom_header")
+                        with gr.Row():
+                            source_after_affine_transforms = gr.Image(
+                                label="Source face after affine augmentations")
+                            source_after_affine_transforms.style(height=240)
+
+                            mask_after_affine_transforms = gr.Image(
+                                label="Mask after elastic augmentations")
+                            mask_after_affine_transforms.style(height=240)
+
+                    with gr.Box():
+                        gr.Markdown("## Step 4", elem_id="custom_header")
+                        gr.Markdown(
+                            "Apply smoothing augmentations to mask for gentle blending", elem_id="custom_header")
+                        mask = gr.Image(label="Mask after Gaussian smoothing")
+                        mask.style(height=240)
+
+                    with gr.Box():
+                        gr.Markdown("## Step 5", elem_id="custom_header")
+                        gr.Markdown(
+                            "Perform blending (based on chosen blending option) to produce self-blended fake", elem_id="custom_header")
+                        with gr.Row():
+                            img_r_before_both_transforms = gr.Image(
+                                label="Real face (for comparison)")
+                            img_r_before_both_transforms.style(height=240)
+
+                            img_f_before_both_transforms = gr.Image(
+                                label="Self-blended fake face")
+                            img_f_before_both_transforms.style(height=240)
+
+                    with gr.Box():
+                        gr.Markdown("## Step 6", elem_id="custom_header")
+                        gr.Markdown(
+                            "Apply the same randomly chosen augmentations to both real and self-blended fake", elem_id="custom_header")
+                        with gr.Row():
+                            img_r_after_both_transforms = gr.Image(
+                                label="Real face after augmentations")
+                            img_r_after_both_transforms.style(height=240)
+
+                            img_f_after_both_transforms = gr.Image(
+                                label="Self-blended fake face after augmentations")
+                            img_f_after_both_transforms.style(height=240)
+                        gr.Markdown(
+                            "Augmentations applied here include RGBShift, HueSaturationValue, RandomBrightnessContrast, ImageCompression from Albumentations.")
+
+                    with gr.Box():
+                        gr.Markdown("## Step 7", elem_id="custom_header")
+                        gr.Markdown(
+                            "Crop real and self-blended fake to only have the faces", elem_id="custom_header")
+                        with gr.Row():
+                            output_r = gr.Image(label="Final real face")
+                            output_r.style(height=240)
+
+                            output_f = gr.Image(label="Final SBI face")
+                            output_f.style(height=240)
+                        gr.Markdown(
+                            "These are the images that are eventually fed into the model for training", elem_id="custom_header")
+
+    with gr.Tab("Consistency Augmentations"):
+        gr.Markdown("Input an image with a face to visualize the consistency augmentations. Values for augmentations are randomly chosen.", elem_id="custom_header")
+        with gr.Row():
+            with gr.Box():
+                gr.Markdown("## Input", elem_id="custom_header")
+                input_image_ca = gr.Image(label="Input Image")
+                input_image_ca.style(height=240)
+                btn_ca = gr.Button(value="Submit")
+                btn_ca.style(full_width=True)
+                gr.Examples(
+                    examples=examples_sbi,
+                    inputs=input_image_ca,
+                    fn=generate_ca,
+                    cache_examples=False,
+                )
+        with gr.Row():
+            with gr.Box():
+                with gr.Row():
+                    og = gr.Image(label="Cropped Face (No augs)")
+                    og.style(height=240)
+                    re = gr.Image(label="Random Erasing")
+                    re.style(height=240)
+                    rc = gr.Image(label="Random Cropping")
+                    rc.style(height=240)
+                    dfdc = gr.Image(label="DFDC Selimsef")
+                    dfdc.style(height=240)
+
+    btn.click(predict_image, inputs=[input_image, model_selection], outputs=[
+              label_probs, output_image])
+    btn_video.click(predict_video, inputs=[input_video, model_selection_video], outputs=[
+                    label_probs_video, output_image_video])
+    btn_sbi.click(generate_sbi, inputs=[input_image_sbi, blending_type, face_region], outputs=[cropped_input_face, img_r_before_both_transforms, img_f_before_both_transforms,
+                                                                                               img_r_after_both_transforms, img_f_after_both_transforms, output_r, output_f, mask, 
+                                                                                               mask_original, source_before_affine_transforms, source_after_affine_transforms, mask_after_affine_transforms])
+    btn_ca.click(generate_ca, inputs=[
+                 input_image_ca], outputs=[og, re, rc, dfdc])
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,8 @@
+dlib
+retinaface_pytorch
+imutils
+numpy
+grad-cam
+gradio 
+opencv-python
+efficientnet_pytorch