Face ID was a reverse engineering challenge during L3AK CTF in 2025. Out of 1587 teams, 6 teams were able to solve it during the CTF. This writeup explains how I solved it. If you would like to try the challenge yourself, you can download it here.
The challenge implements a facial recognition system. If the user provides an image that passes all the checks, the user obtains the flag. The details are described below.
Initial Analysis
As always, we load the binary into IDA to figure out how it works. The binary has a statically linked C library, but we can guess the purpose of most functions based on how they are used. It is probably also possible to port libc symbols from the library into the challenge by creating a database of function signatures, but I did not try this during the CTF.
The following is a rough decompilation of the main logic behind the program:
int run(bool registration_enabled) {
uint8_t headerbuf[100];
uint8_t linebuf[7500];
uint8_t pixelbuf[750000];
uint8_t weightbuf[250000];
int user_rows[500];
int user_cols[500];
int *db_rows;
int *db_cols;
puts("Please input an image in PPM format (ascii)");
memset(headerbuf, 0, sizeof(headerbuf));
fgets(headerbuf, 5, stdin);
if (memcmp(headerbuf, "P3", 2)) {
puts("Wrong header! Please use ppm");
return -1;
}
fgets(headerbuf, 10, stdin);
if (memcmp(headerbuf, "500 500", 7)) {
puts("Sorry, only 500 by 500 images are supported");
return -1;
}
memset(pixelbuf, 0, sizeof(pixelbuf));
memset(linebuf, 0, sizeof(linebuf));
for (int i = 0; i < 500; i++) {
fgets(linebuf, 7500, stdin);
parse_line(linebuf, pixelbuf, i);
}
memset(weightbuf, 0, sizeof(weightbuf));
int index = 0;
for (int y = 0; y < 500; y++) {
for (int x = 0; x < 500; x++) {
int dx = 28900 * (x - 250) * (x - 250);
int dy = 20736 * (y - 250) * (y - 250);
if (dx + dy > 598790400 && y > 150) {
int r = pixelbuf[3 * (500 * y + x)];
int g = pixelbuf[3 * (500 * y + x) + 1];
int r = pixelbuf[3 * (500 * y + x) + 2];
weightbuf[index++] = (r + g + b) / 3;
}
}
}
sort_array(weightbuf, index, 1, compare_weights);
int middle_index = (index - 1) / 2 - 1;
int first_quartile = weightbuf[middle_index / 2];
int third_quartile = weightbuf[middle_index / 2 + (middle_index | 1)];
int range = third_quartile - first_quartile;
int low_threshold = first_quartile - (range + range / 2);
int high_threshold = third_quartile + (range + range / 2);
if (low_threshold < 0) {
low_threshold = 0;
}
memset(user_rows, 0, sizeof(user_rows));
memset(user_cols, 0, sizeof(user_cols));
for (int y = 0; y < 500; y++) {
for (int x = 0; x < 500; x++) {
int r = pixelbuf[3 * (500 * y + x)];
int g = pixelbuf[3 * (500 * y + x) + 1];
int r = pixelbuf[3 * (500 * y + x) + 2];
int avergae = (r + g + b) / 3;
bool between_threshold = low_threshold < average && average < high_threshold;
user_rows[y] += between_threshold;
user_cols[x] += between_threshold;
}
}
uint64_t width, height;
FILE *file;
int laxness;
if (registration_enabled) {
width = 500;
height = 500;
file = fopen("user.db", "wb");
fwrite(&width, 8, 1, file);
fwrite(user_cols, 4, width, file);
fwrite(&height, 8, 1, file);
fwrite(user_rows, 4, height, file);
ask_for_confirmation();
puts("How lax do you want the login to be? ");
laxness = 0;
scanf("%d", &laxness);
fwrite(&laxness, 4, 1, file);
fclose(file);
puts("Registration successful! Please log in...");
return 0;
}
file = fopen("user.db", "rb");
if (!file) {
perror("fopen");
return -1;
}
fread(&width, 8, 1, file);
db_cols = malloc(4 * width);
fread(db_cols, 4, width, file);
fread(&height, 8, 1, file);
db_rows = malloc(4 * height);
fread(db_rows, 4, height, file);
fread(&laxness, 4, 1, file):
fclose(file);
bool valid = true;
for (int x = 0; x < width; x++) {
int deviation = db_cols[x] - user_cols[x];
if (deviation < 0) {
deviation = -deviation;
}
if (deviation > laxness) {
valid = false;
break;
}
}
for (int y = 0; y < height; y++) {
int deviation = db_rows[y] - user_rows[y];
if (deviation < 0) {
deviation = -deviation;
}
if (deviation > laxness) {
valid = false;
break;
}
}
free(db_cols);
free(db_rows);
if (!valid) {
printf("Login failed: sums out of ±%d range.\n", laxness);
return -1;
}
FILE *flag = fopen("flag.txt", "r");
while (true) {
char c = getc(flag);
if (c == -1) {
break;
}
putchar(c);
}
fclose(flag);
fflush(stdout);
return 0;
}
int main() {
puts("Registration disabled, defaulting to login...");
run(false);
return 0;
}
Although the program implements registration as well, this is always disabled on the server. We mostly care about the login function.
First, the program reads a 500x500 pixel image in PPM format from stdin. The pixels are stored in a buffer in RGB format. Then, for a certain oval area, the program calculates the brightness of the pixels and stores the result in another buffer. The brightness is determined by taking the average of the R, G and B values, and the area is defined by the following formula:
def is_calibration(x, y):
dx = (170 * (x - 250)) ** 2
dy = (144 * (y - 250)) ** 2
return y > 150 and dx + dy > 598790400 and dx + dy < 1 << 31
The dx + dy < 1 << 31 constraint is not actually visible in the code. The reason that dx + dy must be smaller than 1 << 31 is that their sum is stored in a 32-bit integer. A value that is larger than 1 << 31 would cause an integer overflow and become a negative number instead.
Next, the buffer with the brightness values is sorted. A minimum and maximum threshold are determined from its quartiles. For every row and column of the image, the program calculates how many pixels are between the thresholds of the previous step. Essentially, the program determines the number of outliers in each row and column of the image. Finally, the number of outliers is compared against the values are defined in user.db. If in every row and column, the number of outliers differs from the expected value by at most 20, the flag is revealed to the user.
Breaking the Algorithm
The goal is to generate an image that satisfies the above constraints. In the end, my solution consisted of two major steps: deciding which pixels will be between the minimum and maximum threshold values (i.e. generating a map of outliers) and finding brightness values that generate the desired thresholds.
Remember that the threshold values depend on the pixels in a certain area. I'll call this area the calibration area. To decide which pixels we would like to place between the threshold values, I started out by assuming that all pixels in the calibration area are within the threshold values, and all other pixels are not. This will make it easier to control the threshold values later on.
Visualized as an image, the calibration area is the white area in the following image:
This is our initial outlier map. Next, my solution modifies the outlier map until the number of outliers in each row and column differs from the values in user.db by at most 20. To achieve this my solution loops through each row in the map, and either enables or disables outliers until the number of outliers matches the desired value. The pixels that are changed are chosen randomly, but with a fixed seed for reproduction. It then does the same for each column. Now, some of the rows may be broken again after fixing the columns. Therefore, my solution alternatingly loops through all rows and all columns until all rows and columns differ from the desired values by at most 20. After a few iterations, this is the case. The following images display how each step of the algorithm affects our outlier map:
Choosing Brightness Values
The last image in the table above shows which pixels we want to place between the threshold values. All other pixels should be outside of this range.
In the end, I solved it with a bit of trial and error. I decided to use the brightness values 0, 124, 132 and 255. If the first quartile of the calibration area is 124, and the third quartile is 132, then the threshold values will be set to 112 and 144. In this case, the brightness values 124 and 132 would fall between the threshold values, and the values 0 and 255 would not.
We can get these quartiles by giving half of the outliers the brightness value 0 and the other half the value 255, and by giving half of the non-outliers the value 124 and the other half the value 132. In my solution script, I simply randomly assigned each pixel one of the two possible values. This led to the following image:
After saving this image in PPM format, and passing it to the challenge via stdin, I obtained the flag!
Implementation
The following script implements the complete solution. Executing this script requires the anynet package, which can be installed with pip install anynet. This package provides the stream class that I used to parse the user.db file.
from PIL import Image
from anynet import streams
import random
def save_image(pixels):
im = Image.new("RGBA", (500, 500))
for y in range(500):
for x in range(500):
p = pixels[y][x]
im.putpixel((x, y), (p, p, p))
im.save("image.png")
def save_ppm(pixels):
lines = ["P3", "500 500"]
for y in range(500):
line = ""
for x in range(500):
p = pixels[y][x]
line += f"{p} {p} {p} "
lines.append(line)
with open("image.ppm", "w") as f:
f.write("\n".join(lines))
def save_map(map, filename):
im = Image.new("RGBA", (500, 500))
for y in range(500):
for x in range(500):
p = map[y][x] * 255
im.putpixel((x, y), (p, p, p))
im.save(filename)
def is_calibration(x, y):
dx = (170 * (x - 250)) ** 2
dy = (144 * (y - 250)) ** 2
return y > 150 and dx + dy > 598790400 and dx + dy < 1 << 31
def random_map():
map = []
for y in range(500):
row = []
for x in range(500):
row.append(random.choice([0, 1]))
map.append(row)
return map
def initial_map():
map = []
for y in range(500):
row = []
for x in range(500):
row.append(is_calibration(x, y))
map.append(row)
return map
def count_map(map):
rowcount = [0] * 500
colcount = [0] * 500
for y in range(500):
for x in range(500):
rowcount[y] += map[y][x]
colcount[x] += map[y][x]
return rowcount, colcount
def find_errors(map):
rowcount, colcount = count_map(map)
rowdiff = []
coldiff = []
for i in range(500):
rowdiff.append(rowcount[i] - horz[i])
coldiff.append(colcount[i] - vert[i])
return rowdiff, coldiff
def check_map(map):
rowdiff, coldiff = find_errors(map)
return all(abs(x) <= 20 for x in rowdiff) and all(abs(x) <= 20 for x in coldiff)
def evaluate_map(map):
rowdiff, coldiff = find_errors(map)
print(sum(abs(x) for x in rowdiff), sum(abs(x) for x in coldiff))
def fix_map_rows(map):
rowdiff, coldiff = find_errors(map)
for y in range(500):
count = abs(rowdiff[y])
if rowdiff[y] < 0:
search = 0
replace = 1
else:
search = 1
replace = 0
positions = [x for x in range(500) if map[y][x] == search]
calib = [x for x in positions if is_calibration(x, y)]
nocalib = [x for x in positions if not is_calibration(x, y)]
random.shuffle(calib)
random.shuffle(nocalib)
positions = nocalib + calib
for i in range(count):
map[y][positions[i]] = replace
def fix_map_columns(map):
rowdiff, coldiff = find_errors(map)
for x in range(500):
count = abs(coldiff[x])
if coldiff[x] < 0:
search = 0
replace = 1
else:
search = 1
replace = 0
positions = [y for y in range(500) if map[y][x] == search]
calib = [y for y in positions if is_calibration(x, y)]
nocalib = [y for y in positions if not is_calibration(x, y)]
random.shuffle(calib)
random.shuffle(nocalib)
positions = nocalib + calib
for i in range(count):
map[positions[i]][x] = replace
def generate_map():
map = initial_map()
save_map(map, "map_initial.png")
index = 0
while not check_map(map):
fix_map_rows(map)
fix_map_columns(map)
save_map(map, "map_fixed.png")
return map
with open("user.db", "rb") as f:
data = f.read()
stream = streams.StreamIn(data, "<")
vert = stream.repeat(stream.u32, stream.u64())
horz = stream.repeat(stream.u32, stream.u64())
threshold = stream.u32()
random.seed(0)
map = generate_map()
pixels = []
for y in range(500):
row = []
for x in range(500):
if map[y][x]:
row.append(random.choice([124, 132]))
else:
row.append(random.choice([0, 255]))
pixels.append(row)
save_image(pixels)
save_ppm(pixels)