--- a/src/libgcrypt-config.in
+++ b/src/libgcrypt-config.in
@@ -166,9 +166,12 @@ if test "$echo_libs" = "yes"; then
     libs_final="$libs"
 
     # Set up `libdirs'.
-    if test "x$libdir" != "x/usr/lib" && test "x$libdir" != "x/lib"; then
-	libdirs="-L$libdir"
-    fi
+    case "$libdir" in
+        /usr/lib|/usr/lib64|/lib|/lib64) ;;
+        *)
+            libdirs="-L$libdir"
+            ;;
+    esac
 
     # Set up `libs_final'.
     libs_final="$libs_final $gpg_error_libs"