diff --git a/common/rfb/CMakeLists.txt b/common/rfb/CMakeLists.txt
index 56d0da6..67dd669 100644
--- a/common/rfb/CMakeLists.txt
+++ b/common/rfb/CMakeLists.txt
@@ -97,8 +97,6 @@ if(GNUTLS_FOUND)
   )
 endif()
 
-add_library(rfb STATIC ${RFB_SOURCES})
-
 # SSE2
 
 set(SSE2_SOURCES
@@ -108,20 +106,20 @@ set(SCALE_DUMMY_SOURCES
   scale_dummy.cxx)
 
 if(COMPILER_SUPPORTS_SSE2)
-  add_library(scale_sse2 STATIC ${SSE2_SOURCES})
-  set(RFB_LIBRARIES
-    ${RFB_LIBRARIES}
-    scale_sse2
+  set_source_files_properties(${SSE2_SOURCES} PROPERTIES COMPILE_FLAGS ${COMPILE_FLAGS} -msse2)
+  set(RFB_SOURCES
+    ${RFB_SOURCES}
+    ${SSE2_SOURCES}
   )
-  set_target_properties(scale_sse2 PROPERTIES COMPILE_FLAGS ${COMPILE_FLAGS} -msse2)
 else()
-  add_library(scale_dummy STATIC ${SCALE_DUMMY_SOURCES})
-  set(RFB_LIBRARIES
-    ${RFB_LIBRARIES}
-    scale_dummy
+  set(RFB_SOURCES
+    ${RFB_SOURCES}
+    ${SCALE_DUMMY_SOURCES}
   )
 endif()
 
+add_library(rfb STATIC ${RFB_SOURCES})
+
 target_link_libraries(rfb ${RFB_LIBRARIES})
 
 if(UNIX)