ekoparty Security Conference 6 - Challenge ESET 2010

Challenge ESET 2010
ESET Challenge is a multiple levels crackme challenge, the higher score you obtained the better chances you will have of accessing to the discount.
The delivery would consist of a compressed file in RAR format, containing all the demands in the instructions. All text documents must be in TXT format.

.: NAME UPPERCASE/INFERIORITY ROUTINE :.

00401383  |.  8A06            MOV AL,BYTE PTR DS:[ESI]         ; | MOVE ESI [NAME] TO AL
00401385  |.  84C0            TEST AL,AL                       ; | There is no char to check ? so AL = 00, = Jump
00401387  |.  74 13           JE SHORT 0040139C                ; | Jump if AL = 00 (no chars to check) so we leave the routine, and call the calculation routine
00401389  |.  3C 41           CMP AL,41                        ; | Watch if the char is inferior to the hex: 41 = A
0040138B  |.  72 1F           JB SHORT 004013AC                ; | Then Bad boy "Intentalo de nuevo"
0040138D  |.  3C 5A           CMP AL,5A                        ; | Watch if the char is superior to the hex: 5A = Z
0040138F  |.  73 03           JNB SHORT 00401394               ; | Then CALL 004013D2 (mean the char is not a A-Z uppercase letter)
00401391  |.  46              INC ESI                          ; | Increment a char
00401392  |.  EB EF           JMP SHORT 00401383               ; | Return at the top of the routine
00401394  |.  E8 39000000     CALL 004013D2                    ; | CALL the uppercase transformation
00401399  |.  46              INC ESI                          ; | Increment a char
0040139A  |.  EB E7           JMP SHORT 00401383               ; | Return at the top of the routine


004013D2  |.  2C 20           SUB AL,20                        ; | Make the char uppercase
004013D4  |.  8806            MOV BYTE PTR DS:[ESI],AL         ; | Move the maked uppercase char in esi
004013D6  |.  C3              RETN                             ; | Return on to 00401399


.: CALCULATION ROUTINE :.

004013C2  |.  33FF            XOR EDI,EDI                      ; | Make edi = 00000000
004013C4  |.  33DB            XOR EBX,EBX                      ; | Make ebx = 00000000
004013C6  |.  8A1E            MOV BL,BYTE PTR DS:[ESI]         ; | Move the first char in BL
004013C8  |.  84DB            TEST BL,BL                       ; | There is no char to check ? so AL = 00, = Jump
004013CA  |.  74 05           JE SHORT 004013D1                ; | Jump if AL = 00 (no chars for continue the addition) so we leave the routine
004013CC  |.  03FB            ADD EDI,EBX                      ; | Make an addition with your hex char (in EBX) and EDI (the result will be in EDI)
004013CE  |.  46              INC ESI                          ; | Increment a char
004013CF  |.  EB F5           JMP SHORT 004013C6               ; | Return at the top of the routine
my name is XYLITOL
XYLITOL = (58 59 4C 49 54 4F 4C)
58 + 0 = 58 | 59 + 58 = B1 | 4C + B1 = FD | 49 + FD = 146 | 54 + 146 = 19A | 4F + 19A = 1E9 | 4c + 1E9 = 235


004013A2  |.  81F7 78560000   XOR EDI,5678                     ; | XOR EDI (the result of the addition) to 5678
004013A8  |.  8BC7            MOV EAX,EDI                      ; | Move (your result) in EAX
004013AA  |.  EB 15           JMP SHORT 004013C1               ; | Jump
XOR for me: XYLITOL = 235 XOR 5678 = 544D


00401232  |.  50              PUSH EAX                         ; | Add the xor result (544D for me) in the stack
00401233  |.  68 7E214000     PUSH 40217E                      ; | Add your entered serial in the stack
00401238  |.  E8 9B010000     CALL 004013D8                    ; | CALL the serial check verification



004013D8  |.  33C0            XOR EAX,EAX                      ; | Make eax = 00000000
004013DA  |.  33FF            XOR EDI,EDI                      ; | Make edi = 00000000
004013DC  |.  33DB            XOR EBX,EBX                      ; | Make ebx = 00000000
004013DE  |.  8B7424 04       MOV ESI,DWORD PTR SS:[ESP+4]     ; | Move the entered serial in esi
004013E2  |.  B0 0A           MOV AL,0A                        ; | MOVE 0A TO AL [WATCH EAX]
004013E4  |.  8A1E            MOV BL,BYTE PTR DS:[ESI]         ; | MOVE ESI[OUR ENTERED SERiAL] TO BL [WATCH EBX]
004013E6  |.  84DB            TEST BL,BL                       ; | There is no char to check ? so AL = 00, = Jump
004013E8  |.  74 0B           JE SHORT 004013F5                ; | Jump if AL = 00 (no chars to check)
004013EA  |.  80EB 30         SUB BL,30                        ; | BL - 30
004013ED  |.  0FAFF8          IMUL EDI,EAX                     ; | EDI = EDI * EAX [REMEMBER THAT EAX=0A]
004013F0  |.  03FB            ADD EDI,EBX                      ; | Make an addition =, EDI + EAX
004013F2  |.  46              INC ESI                          ; | Increment a char
004013F3  |.  EB ED           JMP SHORT 004013E2               ; | Return at the top of the routine


004013F5  |.  81F7 34120000   XOR EDI,1234                     ; | EDI^1234
004013FB  |.  8BDF            MOV EBX,EDI                      ; | Move edi in ebx
004013FD  |.  C3              RETN                             ; | Return to 0040123D



.: FINAL CHECK :.

0040123D  |.  83C4 04         ADD ESP,4                        ; | add +4 in esp (useless for us)
00401240  |.  58              POP EAX                          ; | Take the serial calculated (544D for me)
00401241  |.  3BC3            CMP EAX,EBX                      ; | Compare eax to ebx (544D to ???)
00401243  |.  74 07           JE SHORT 0040124C                ; | jump if equal
00401245  |.  E8 18010000     CALL 00401362                    ; | Bad boy
0040124A  |.  EB 9A           JMP SHORT 004011E6               ; | End
0040124C  |.  E8 FC000000     CALL 0040134D                    ; | Good boy
00401251  |.  EB 93           JMP SHORT 004011E6               ; | End

KeyGen:
asm file

.486
.model  flat, stdcall
option  casemap :none   ; case sensitive

include     base.inc

.data
format db "%d",0

.data?
Namebuffer dd 512 dup(?)
serial dd 512 dup(?)

.code
start:
    invoke  GetModuleHandle, NULL
    mov hInstance, eax
    invoke  DialogBoxParam, hInstance, 101, 0, ADDR DlgProc, 0
    invoke  ExitProcess, eax
; -----------------------------------------------------------------------
DlgProc proc    hWin    :DWORD,
        uMsg    :DWORD,
        wParam  :DWORD,
        lParam  :DWORD
    .if uMsg == WM_COMMAND
        .if wParam == IDC_OK
; -----------------------------------------------------------------------
invoke GetDlgItemText,hWin,1001,addr Namebuffer,sizeof Namebuffer
invoke CharUpper,addr Namebuffer
  lea esi,Namebuffer
  XOR EDI,EDI
  XOR EBX,EBX
L002:
  MOV BL,BYTE PTR DS:[esi]
  TEST BL,BL
  JE SHORT @xor
  ADD EDI,EBX
  INC ESI
  JMP L002
  @xor:
  xor edi,5678h
  xor edi,1234h
  invoke wsprintf,addr serial,addr format,edi
  invoke SetDlgItemText,hWin,1002,addr serial
  ret
; -----------------------------------------------------------------------
        .elseif wParam == IDC_IDCANCEL
            invoke EndDialog,hWin,0
        .endif
    .elseif uMsg == WM_CLOSE
        invoke  EndDialog,hWin,0
    .endif
    xor eax,eax
    ret
DlgProc endp
end start

inc file

include windows.inc

uselib  MACRO   libname
    include     libname.inc
    includelib  libname.lib
ENDM

uselib  user32
uselib  kernel32

DlgProc     PROTO :DWORD,:DWORD,:DWORD,:DWORD
IDC_OK          equ 1003
IDC_IDCANCEL    equ 1004

.data?
hInstance       dd      ?   ;dd can be written as dword

rc file

;This Resource Script was generated by WinAsm Studio.

#define IDC_OK 1003
#define IDC_CANCEL 1004

101 DIALOGEX 0,0,169,44
CAPTION "ESET KeyGen"
FONT 8,"Tahoma"
STYLE 0x80c80880
EXSTYLE 0x00000000

BEGIN
    CONTROL "Generate",IDC_OK,"Button",0x10000001,110,5,50,14,0x00000000
    CONTROL "Exit",IDC_CANCEL,"Button",0x10000000,110,23,50,14,0x00000000
    CONTROL "",1001,"Edit",0x10000080,7,6,90,12,0x00000200
    CONTROL "",1002,"Edit",0x10000080,7,24,90,12,0x00000200
END

Edit: time to remake the keygen better :)